mirror of
https://github.com/nickspaargaren/no-google.git
synced 2023-12-13 21:00:26 +01:00
Update regex install script (#160)
* Update readme * Update install script
This commit is contained in:
parent
3cba4d6150
commit
580d5aeff0
3 changed files with 233 additions and 264 deletions
|
@ -59,7 +59,8 @@ NOTE: Youtube advertisements are pretty difficult to block trough DNS, as they m
|
|||
|
||||
You can also easily use the modified [pihole regex installer script](https://github.com/mmotti/pihole-regex) by [@mmoti](https://github.com/mmotti) by executing this terminal command from your raspberry Pi Pi-hole server.
|
||||
```
|
||||
curl -sSl https://raw.githubusercontent.com/nickspaargaren/no-google/master/install.sh | bash
|
||||
curl -sSl https://raw.githubusercontent.com/nickspaargaren/no-google/master/install.py | sudo python3
|
||||
|
||||
```
|
||||
and then, executing it. It should add all of the above regex automatically.
|
||||
|
||||
|
|
231
install.py
Normal file
231
install.py
Normal file
|
@ -0,0 +1,231 @@
|
|||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
|
||||
def fetch_url(url):
|
||||
|
||||
if not url:
|
||||
return
|
||||
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0'}
|
||||
|
||||
print('[i] Fetching:', url)
|
||||
|
||||
try:
|
||||
response = urlopen(Request(url, headers=headers))
|
||||
except HTTPError as e:
|
||||
print('[E] HTTP Error:', e.code, 'whilst fetching', url)
|
||||
return
|
||||
except URLError as e:
|
||||
print('[E] URL Error:', e.reason, 'whilst fetching', url)
|
||||
return
|
||||
|
||||
# Read and decode
|
||||
response = response.read().decode('UTF-8').replace('\r\n', '\n')
|
||||
|
||||
# If there is data
|
||||
if response:
|
||||
# Strip leading and trailing whitespace
|
||||
response = '\n'.join(x for x in map(str.strip, response.splitlines()))
|
||||
|
||||
# Return the hosts
|
||||
return response
|
||||
|
||||
|
||||
url_regexps_remote = 'https://raw.githubusercontent.com/nickspaargaren/no-google/master/regex.list'
|
||||
install_comment = 'github.com/nickspaargaren/no-google'
|
||||
|
||||
cmd_restart = ['pihole', 'restartdns', 'reload']
|
||||
|
||||
db_exists = False
|
||||
conn = None
|
||||
c = None
|
||||
|
||||
regexps_remote = set()
|
||||
regexps_local = set()
|
||||
regexps_mmotti_local = set()
|
||||
regexps_legacy_mmotti = set()
|
||||
regexps_remove = set()
|
||||
|
||||
# Start the docker directory override
|
||||
print('[i] Checking for "pihole" docker container')
|
||||
|
||||
# Initialise the docker variables
|
||||
docker_id = None
|
||||
docker_mnt = None
|
||||
docker_mnt_src = None
|
||||
|
||||
# Check to see whether the default "pihole" docker container is active
|
||||
try:
|
||||
docker_id = subprocess.run(['docker', 'ps', '--filter', 'name=pihole', '-q'],
|
||||
stdout=subprocess.PIPE, universal_newlines=True).stdout.strip()
|
||||
# Exception for if docker is not installed
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
# If a pihole docker container was found, locate the first mount
|
||||
if docker_id:
|
||||
docker_mnt = subprocess.run(['docker', 'inspect', '--format', '{{ (json .Mounts) }}', docker_id],
|
||||
stdout=subprocess.PIPE, universal_newlines=True).stdout.strip()
|
||||
# Convert output to JSON and iterate through each dict
|
||||
for json_dict in json.loads(docker_mnt):
|
||||
# If this mount's destination is /etc/pihole
|
||||
if json_dict['Destination'] == r'/etc/pihole':
|
||||
# Use the source path as our target
|
||||
docker_mnt_src = json_dict['Source']
|
||||
break
|
||||
|
||||
# If we successfully found the mount
|
||||
if docker_mnt_src:
|
||||
print('[i] Running in docker installation mode')
|
||||
# Prepend restart commands
|
||||
cmd_restart[0:0] = ['docker', 'exec', '-i', 'pihole']
|
||||
else:
|
||||
print('[i] Running in physical installation mode ')
|
||||
|
||||
# Set paths
|
||||
path_pihole = docker_mnt_src if docker_mnt_src else r'/etc/pihole'
|
||||
path_legacy_regex = os.path.join(path_pihole, 'regex.list')
|
||||
path_legacy_mmotti_regex = os.path.join(path_pihole, 'mmotti-regex.list')
|
||||
path_pihole_db = os.path.join(path_pihole, 'gravity.db')
|
||||
|
||||
# Check that pi-hole path exists
|
||||
if os.path.exists(path_pihole):
|
||||
print('[i] Pi-hole path exists')
|
||||
else:
|
||||
print(f'[e] {path_pihole} was not found')
|
||||
exit(1)
|
||||
|
||||
# Check for write access to /etc/pihole
|
||||
if os.access(path_pihole, os.X_OK | os.W_OK):
|
||||
print(f'[i] Write access to {path_pihole} verified')
|
||||
else:
|
||||
print(f'[e] Write access is not available for {path_pihole}. Please run as root or other privileged user')
|
||||
exit(1)
|
||||
|
||||
# Determine whether we are using DB or not
|
||||
if os.path.isfile(path_pihole_db) and os.path.getsize(path_pihole_db) > 0:
|
||||
db_exists = True
|
||||
print('[i] DB detected')
|
||||
else:
|
||||
print('[i] Legacy regex.list detected')
|
||||
|
||||
# Fetch the remote regexps
|
||||
str_regexps_remote = fetch_url(url_regexps_remote)
|
||||
|
||||
# If regexps were fetched, remove any comments and add to set
|
||||
if str_regexps_remote:
|
||||
regexps_remote.update(x for x in map(str.strip, str_regexps_remote.splitlines()) if x and x[:1] != '#')
|
||||
print(f'[i] {len(regexps_remote)} regexps collected from {url_regexps_remote}')
|
||||
else:
|
||||
print('[i] No remote regexps were found.')
|
||||
exit(1)
|
||||
|
||||
if db_exists:
|
||||
# Create a DB connection
|
||||
print(f'[i] Connecting to {path_pihole_db}')
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(path_pihole_db)
|
||||
except sqlite3.Error as e:
|
||||
print(e)
|
||||
exit(1)
|
||||
|
||||
# Create a cursor object
|
||||
c = conn.cursor()
|
||||
|
||||
# Add / update remote regexps
|
||||
print('[i] Adding / updating regexps in the DB')
|
||||
|
||||
c.executemany('INSERT OR IGNORE INTO domainlist (type, domain, enabled, comment) '
|
||||
'VALUES (3, ?, 1, ?)',
|
||||
[(x, install_comment) for x in sorted(regexps_remote)])
|
||||
c.executemany('UPDATE domainlist '
|
||||
'SET comment = ? WHERE domain in (?) AND comment != ?',
|
||||
[(install_comment, x, install_comment) for x in sorted(regexps_remote)])
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Fetch all current mmotti regexps in the local db
|
||||
c.execute('SELECT domain FROM domainlist WHERE type = 3 AND comment = ?', (install_comment,))
|
||||
regexps_mmotti_local_results = c.fetchall()
|
||||
regexps_mmotti_local.update([x[0] for x in regexps_mmotti_local_results])
|
||||
|
||||
# Remove any local entries that do not exist in the remote list
|
||||
# (will only work for previous installs where we've set the comment field)
|
||||
print('[i] Identifying obsolete regexps')
|
||||
regexps_remove = regexps_mmotti_local.difference(regexps_remote)
|
||||
|
||||
if regexps_remove:
|
||||
print('[i] Removing obsolete regexps')
|
||||
c.executemany('DELETE FROM domainlist WHERE type = 3 AND domain in (?)', [(x,) for x in regexps_remove])
|
||||
conn.commit()
|
||||
|
||||
# Delete mmotti-regex.list as if we've migrated to the db, it's no longer needed
|
||||
if os.path.exists(path_legacy_mmotti_regex):
|
||||
os.remove(path_legacy_mmotti_regex)
|
||||
|
||||
print('[i] Restarting Pi-hole')
|
||||
subprocess.run(cmd_restart, stdout=subprocess.DEVNULL)
|
||||
|
||||
# Prepare final result
|
||||
print('[i] Done - Please see your installed regexps below\n')
|
||||
|
||||
c.execute('Select domain FROM domainlist WHERE type = 3')
|
||||
final_results = c.fetchall()
|
||||
regexps_local.update(x[0] for x in final_results)
|
||||
|
||||
print(*sorted(regexps_local), sep='\n')
|
||||
|
||||
conn.close()
|
||||
|
||||
else:
|
||||
# If regex.list exists and is not empty
|
||||
# Read it and add to a set
|
||||
if os.path.isfile(path_legacy_regex) and os.path.getsize(path_legacy_regex) > 0:
|
||||
print('[i] Collecting existing entries from regex.list')
|
||||
with open(path_legacy_regex, 'r') as fRead:
|
||||
regexps_local.update(x for x in map(str.strip, fRead) if x and x[:1] != '#')
|
||||
|
||||
# If the local regexp set is not empty
|
||||
if regexps_local:
|
||||
print(f'[i] {len(regexps_local)} existing regexps identified')
|
||||
# If we have a record of a previous legacy install
|
||||
if os.path.isfile(path_legacy_mmotti_regex) and os.path.getsize(path_legacy_mmotti_regex) > 0:
|
||||
print('[i] Existing mmotti-regex install identified')
|
||||
# Read the previously installed regexps to a set
|
||||
with open(path_legacy_mmotti_regex, 'r') as fOpen:
|
||||
regexps_legacy_mmotti.update(x for x in map(str.strip, fOpen) if x and x[:1] != '#')
|
||||
|
||||
if regexps_legacy_mmotti:
|
||||
print('[i] Removing previously installed regexps')
|
||||
regexps_local.difference_update(regexps_legacy_mmotti)
|
||||
|
||||
# Add remote regexps to local regexps
|
||||
print(f'[i] Syncing with {url_regexps_remote}')
|
||||
regexps_local.update(regexps_remote)
|
||||
|
||||
# Output to regex.list
|
||||
print(f'[i] Outputting {len(regexps_local)} regexps to {path_legacy_regex}')
|
||||
with open(path_legacy_regex, 'w') as fWrite:
|
||||
for line in sorted(regexps_local):
|
||||
fWrite.write(f'{line}\n')
|
||||
|
||||
# Output mmotti remote regexps to mmotti-regex.list
|
||||
# for future install / uninstall
|
||||
with open(path_legacy_mmotti_regex, 'w') as fWrite:
|
||||
for line in sorted(regexps_remote):
|
||||
fWrite.write(f'{line}\n')
|
||||
|
||||
print('[i] Restarting Pi-hole')
|
||||
subprocess.run(cmd_restart, stdout=subprocess.DEVNULL)
|
||||
|
||||
# Prepare final result
|
||||
print('[i] Done - Please see your installed regexps below\n')
|
||||
with open(path_legacy_regex, 'r') as fOpen:
|
||||
for line in fOpen:
|
||||
print(line, end='')
|
263
install.sh
263
install.sh
|
@ -1,263 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
#Credits goes of course to mmoti for his regex list script
|
||||
# Set variables
|
||||
db_gravity='/etc/pihole/gravity.db'
|
||||
file_pihole_regex='/etc/pihole/regex.list'
|
||||
file_mmotti_regex='/etc/pihole/mmotti-regex.list'
|
||||
file_mmotti_remote_regex='https://github.com/nickspaargaren/no-google/blob/master/regex.list'
|
||||
installer_comment='github.com/nickspaargaren/no-google'
|
||||
|
||||
# Determine whether we are using Pi-hole DB
|
||||
if [[ -s "${db_gravity}" ]]; then
|
||||
usingDB=true
|
||||
fi
|
||||
|
||||
# Functions
|
||||
function fetchResults {
|
||||
|
||||
local selection="${1}" action="${2}" queryStr
|
||||
|
||||
# Select * if not set
|
||||
[[ -z "${selection}" ]] && selection='*'
|
||||
|
||||
# Determine course of action
|
||||
case "${action}" in
|
||||
migrated_regexps ) queryStr="Select ${selection} FROM regex WHERE comment='${installer_comment}'";;
|
||||
user_created ) queryStr="Select ${selection} FROM regex WHERE comment IS NULL OR comment!='${installer_comment}'";;
|
||||
current_id ) queryStr="Select ${selection} FROM regex ORDER BY id DESC LIMIT 1";;
|
||||
* ) queryStr="Select ${selection} FROM regex";;
|
||||
esac
|
||||
|
||||
# Execute SQL query
|
||||
sqlite3 ${db_gravity} "${queryStr}" 2>&1
|
||||
|
||||
# Check exit status
|
||||
status="$?"
|
||||
[[ "${status}" -ne 0 ]] && { (>&2 echo '[i] An error occured whilst fetching results'); return 1; }
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function updateDB() {
|
||||
|
||||
local inputData="${1}" action="${2}" queryStr
|
||||
|
||||
# Determine course of action
|
||||
case "${action}" in
|
||||
remove_pre_migrated ) queryStr="DELETE FROM regex WHERE domain in (${inputData})";;
|
||||
remove_migrated ) queryStr="DELETE FROM regex WHERE comment = '${installer_comment}'";;
|
||||
* ) return ;;
|
||||
esac
|
||||
|
||||
# Execute SQL query
|
||||
sudo sqlite3 ${db_gravity} "${queryStr}"
|
||||
|
||||
# Check exit status
|
||||
status="$?"
|
||||
[[ "${status}" -ne 0 ]] && { (>&2 echo '[i] An error occured whilst updating database'); return 1; }
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function generateCSV() {
|
||||
|
||||
# Exit if there is a problem with the remoteRegex string
|
||||
[[ -z "${1}" ]] && exit 1
|
||||
|
||||
local remoteRegex timestamp queryArr file_csv_tmp
|
||||
|
||||
# Set local variables
|
||||
remoteRegex="${1}"
|
||||
timestamp="$(date --utc +'%s')"
|
||||
file_csv_tmp="$(mktemp -p "/tmp" --suffix=".csv")"
|
||||
iteration="$(fetchResults "id" "current_id")"
|
||||
|
||||
# At this point we need to double check that there wasn't an error (rather than no result) when trying to
|
||||
# retrieve the current iteration.
|
||||
status="$?"
|
||||
[[ "${status}" -ne 0 ]] && return 1
|
||||
|
||||
# Create array to hold import string
|
||||
declare -a queryArr
|
||||
|
||||
# Start of processing
|
||||
# If we got the id of the last item in the regex table, iterate once
|
||||
# Otherwise set the iterator to 1
|
||||
[[ -n "${iteration}" ]] && ((iteration++)) || iteration=1
|
||||
|
||||
# Iterate through the remote regexps
|
||||
# So long as the line is not empty, generate the CSV values
|
||||
while read -r regexp; do
|
||||
if [[ -n "${regexp}" ]]; then
|
||||
queryArr+=("${iteration},\"${regexp}\",1,${timestamp},${timestamp},\"${installer_comment}\"")
|
||||
((iteration++))
|
||||
fi
|
||||
done <<< "${remoteRegex}"
|
||||
|
||||
# If our array is populated then output the results to a temporary file
|
||||
if [[ "${#queryArr[@]}" -gt 0 ]]; then
|
||||
printf '%s\n' "${queryArr[@]}" > "${file_csv_tmp}"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Output the CSV path
|
||||
echo "${file_csv_tmp}"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "[i] Fetching mmotti's regexps"
|
||||
# Fetch the remote regex file and remove comment lines
|
||||
mmotti_remote_regex=$(wget -qO - "${file_mmotti_remote_regex}" | grep '^[^#]')
|
||||
# Conditional exit if empty
|
||||
[[ -z "${mmotti_remote_regex}" ]] && { echo '[i] Failed to download pihole-google regex list'; exit 1; }
|
||||
|
||||
echo '[i] Fetching existing regexps'
|
||||
# Conditionally fetch existing regexps depending on
|
||||
# whether the user has migrated to the Pi-hole DB or not
|
||||
if [[ "${usingDB}" == true ]]; then
|
||||
str_regex=$(fetchResults "domain")
|
||||
else
|
||||
str_regex=$(grep '^[^#]' < "${file_pihole_regex}")
|
||||
fi
|
||||
|
||||
# Starting the install process
|
||||
# If we're using the Pi-hole DB
|
||||
if [[ "${usingDB}" == true ]]; then
|
||||
# If we found regexps in the database
|
||||
if [[ -n "${str_regex}" ]] ; then
|
||||
echo '[i] Checking for previous migration'
|
||||
# Check whether this script has previously migrated our regexps
|
||||
db_migrated_regexps=$(fetchResults "domain" "migrated_regexps")
|
||||
# If migration is detected
|
||||
if [[ -n "${db_migrated_regexps}" ]]; then
|
||||
echo '[i] Previous migration detected'
|
||||
# As we have already migrated the user, we need to check
|
||||
# whether the regexps in the database are up-to-date
|
||||
echo '[i] Checking whether updates are required'
|
||||
# Use comm -3 to suppress lines that appear in both files
|
||||
# If there are any results returned, this will quickly tell us
|
||||
# that there are discrepancies
|
||||
updatesRequired=$(comm -3 <(sort <<< "${db_migrated_regexps}") <(sort <<< "${mmotti_remote_regex}"))
|
||||
# Conditional exit if no updates are required
|
||||
[[ -z "${updatesRequired}" ]] && { echo '[i] Local regex filter is already up-to-date'; exit 0; }
|
||||
# Now we know that updates are required, it's easiest to start a fresh
|
||||
echo '[i] Running removal query'
|
||||
# Clear our previously migrated domains from the regex table
|
||||
updateDB "" "remove_migrated"
|
||||
else
|
||||
echo '[i] No previous migration detected'
|
||||
# As we haven't yet migrated, we need to manually remove matches
|
||||
# If we have a local mmotti-regex.list, read from that as it was used on the last install (pre-db)
|
||||
# Otherwise, default to the latest remote copy
|
||||
if [[ -s "${file_mmotti_regex}" ]]; then
|
||||
# Only return regexps in both the regex table and regex file
|
||||
mapfile -t result <<< "$(comm -12 <(sort <<< "${str_regex}") <(sort < "${file_mmotti_regex}"))"
|
||||
|
||||
else
|
||||
# Only return regexps in both the regex table and regex file
|
||||
mapfile -t result <<< "$(comm -12 <(sort <<< "${str_regex}") <(sort <<< "${mmotti_remote_regex}"))"
|
||||
fi
|
||||
# If we have matches in both the regex table and regex file
|
||||
if [[ -n "${result[*]}" ]]; then
|
||||
echo '[i] Forming removal string'
|
||||
# regexstring --> 'regexstring1','regexstring2',
|
||||
# Then remove the trailing comma
|
||||
removalStr=$(printf "'%s'," "${result[@]}" | sed 's/,$//')
|
||||
# If our removal string is not empty (sanity check)
|
||||
if [[ -n "${removalStr}" ]]; then
|
||||
echo '[i] Running removal query'
|
||||
# Remove regexps from the regex table if there are in the
|
||||
# removal string
|
||||
updateDB "${removalStr}" "remove_pre_migrated"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo '[i] No regexps currently exist in the database'
|
||||
fi
|
||||
|
||||
# Create our CSV
|
||||
echo '[i] Generating CSV file'
|
||||
csv_file=$(generateCSV "${mmotti_remote_regex}")
|
||||
|
||||
# Conditional exit
|
||||
if [[ -z "${csv_file}" ]] || [[ ! -s "${csv_file}" ]]; then
|
||||
echo '[i] Error: Generated CSV is empty'; exit 1;
|
||||
fi
|
||||
|
||||
# Construct correct input format for import
|
||||
echo '[i] Importing CSV to DB'
|
||||
printf ".mode csv\\n.import \"%s\" %s\\n" "${csv_file}" "regex" | sudo sqlite3 "${db_gravity}" 2>&1
|
||||
|
||||
# Check exit status
|
||||
status="$?"
|
||||
[[ "${status}" -ne 0 ]] && { echo '[i] An error occured whilst importing the CSV into the database'; exit 1; }
|
||||
|
||||
# Status update
|
||||
echo '[i] Regex import complete'
|
||||
|
||||
# Refresh Pi-hole
|
||||
echo '[i] Refreshing Pi-hole'
|
||||
pihole restartdns reload > /dev/null
|
||||
|
||||
# Remove the old mmotti-regex file
|
||||
[[ -e "${file_mmotti_regex}" ]] && sudo rm -f "${file_mmotti_regex}"
|
||||
|
||||
# Output regexps currently in the DB
|
||||
echo $'\n'
|
||||
echo 'These are your current regexps:'
|
||||
fetchResults "domain" | sed 's/^/ /'
|
||||
else
|
||||
if [[ -n "${str_regex}" ]]; then
|
||||
# Restore config prior to previous install
|
||||
# Keep entries only unique to pihole regex
|
||||
if [[ -s "${file_mmotti_regex}" ]]; then
|
||||
echo "[i] Removing pihole-google regex.list from a previous install"
|
||||
comm -23 <(sort <<< "${str_regex}") <(sort "${file_mmotti_regex}") | sudo tee $file_pihole_regex > /dev/null
|
||||
sudo rm -f "${file_mmotti_regex}"
|
||||
else
|
||||
# In the event that file_mmotti_regex is not available
|
||||
# Match against the latest remote list instead
|
||||
echo "[i] Removing pihole-google regex.list from a previous install"
|
||||
comm -23 <(sort <<< "${str_regex}") <(sort <<< "${mmotti_remote_regex}") | sudo tee $file_pihole_regex > /dev/null
|
||||
fi
|
||||
fi
|
||||
|
||||
# Copy latest regex list to file_mmotti_regex dir
|
||||
echo "[i] Copying remote regex.list to ${file_mmotti_regex}"
|
||||
echo "${mmotti_remote_regex}" | sudo tee "${file_mmotti_regex}" > /dev/null
|
||||
|
||||
# Status update
|
||||
echo "[i] $(wc -l <<< "${mmotti_remote_regex}") regexps found in mmotti's regex.list"
|
||||
|
||||
# If pihole regex is not empty after changes
|
||||
if [[ -s "${file_pihole_regex}" ]]; then
|
||||
# Extract non mmotti-regex entries
|
||||
existing_regex_list="$(grep '^[^#]' < "${file_pihole_regex}")"
|
||||
# Form output (preserving existing config)
|
||||
echo "[i] $(wc -l <<< "$existing_regex_list") regexps exist outside of mmotti's regex.list"
|
||||
final_regex=$(printf "%s\n" "${mmotti_remote_regex}" "${existing_regex_list}")
|
||||
else
|
||||
echo "[i] No regex.list differences to mmotti's regex.list"
|
||||
final_regex=$(printf "%s\n" "$mmotti_remote_regex")
|
||||
fi
|
||||
|
||||
# Output to regex.list
|
||||
echo "[i] Saving to ${file_pihole_regex}"
|
||||
LC_COLLATE=C sort -u <<< "${final_regex}" | sudo tee $file_pihole_regex > /dev/null
|
||||
|
||||
# Refresh Pi-hole
|
||||
echo "[i] Refreshing Pi-hole"
|
||||
pihole restartdns reload > /dev/null
|
||||
|
||||
echo "[i] Done"
|
||||
|
||||
# Output to user
|
||||
echo $'\n'
|
||||
cat $file_pihole_regex
|
||||
fi
|
||||
|
||||
exit 0
|
Loading…
Reference in a new issue