From 55f863cd5789248fd2eb064cf10d78b785d535fa Mon Sep 17 00:00:00 2001
From: Nick Spaargaren <nickspaargaren@users.noreply.github.com>
Date: Thu, 30 Dec 2021 11:36:01 +0100
Subject: [PATCH] Add check for duplicate lines workflow (#98)

---
 .github/workflows/duplicate-checker.yml | 27 +++++++++++++++++++++++++
 scripts/test_duplicates.py              | 27 +++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 .github/workflows/duplicate-checker.yml
 create mode 100644 scripts/test_duplicates.py

diff --git a/.github/workflows/duplicate-checker.yml b/.github/workflows/duplicate-checker.yml
new file mode 100644
index 0000000..748cae1
--- /dev/null
+++ b/.github/workflows/duplicate-checker.yml
@@ -0,0 +1,27 @@
+name: Duplicate checker
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'pihole-google.txt'
+  pull_request:
+    branches:
+      - master
+      - develop
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v2
+      name: Python setup
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install pytest
+      run: pip install pytest
+    - name: Check for duplicate lines
+      run: |
+        cd scripts
+        pytest
\ No newline at end of file
diff --git a/scripts/test_duplicates.py b/scripts/test_duplicates.py
new file mode 100644
index 0000000..0e69ce8
--- /dev/null
+++ b/scripts/test_duplicates.py
@@ -0,0 +1,27 @@
+counts = { }
+with open("../pihole-google.txt") as f: 
+    for line in f:
+        stripline = line.strip() # strip whitespace
+        myhash = hash(stripline) 
+        if myhash:
+            if myhash in counts: # duplicate line, inc count
+                counts[myhash] = counts[myhash]+1 
+            else:
+                counts[myhash] = 1 # new entry 
+f.close()
+#re-read file, and print out duplicate lines 
+with open("../pihole-google.txt") as f: 
+    for line in f:
+        stripline = line.strip() 
+        myhash = hash(stripline) 
+        if myhash:
+            if counts[myhash]>1: 
+                # print duplicate line and count
+                assert False, stripline + " occurred more than one time in pihole-google.txt, please remove duplicate domains."
+                # after printing dup, clear ctr so prints once
+                counts[myhash] = 0
+
+f.close()
+
+def test_succes():
+    assert(True)
\ No newline at end of file