Add check for duplicate lines workflow (#98)

.github/workflows/duplicate-checker.yml

@@ -0,0 +1,27 @@
+name: Duplicate checker
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'pihole-google.txt'
+  pull_request:
+    branches:
+      - master
+      - develop
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v2
+      name: Python setup
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install pytest
+      run: pip install pytest
+    - name: Check for duplicate lines
+      run: |
+        cd scripts
+        pytest

scripts/test_duplicates.py

@@ -0,0 +1,27 @@
+counts = { }
+with open("../pihole-google.txt") as f: 
+    for line in f:
+        stripline = line.strip() # strip whitespace
+        myhash = hash(stripline) 
+        if myhash:
+            if myhash in counts: # duplicate line, inc count
+                counts[myhash] = counts[myhash]+1 
+            else:
+                counts[myhash] = 1 # new entry 
+f.close()
+#re-read file, and print out duplicate lines 
+with open("../pihole-google.txt") as f: 
+    for line in f:
+        stripline = line.strip() 
+        myhash = hash(stripline) 
+        if myhash:
+            if counts[myhash]>1: 
+                # print duplicate line and count
+                assert False, stripline + " occurred more than one time in pihole-google.txt, please remove duplicate domains."
+                # after printing dup, clear ctr so prints once
+                counts[myhash] = 0
+
+f.close()
+
+def test_succes():
+    assert(True)