Moved file management-related functions to separate module 'files', introduced global file caching
This commit is contained in:
faildev_mode
parent
5fc8db87d4
commit
f86deb1fee
|
|
@ -1,6 +1,6 @@
|
|||
import os
|
||||
import urllib.parse
|
||||
from item import Item
|
||||
from item import Item # from itself
|
||||
|
||||
class Content:
|
||||
"""This class contains some functions useful for evaluation of Python code in content files"""
|
||||
|
|
|
|||
114
_src/build.py
114
_src/build.py
|
|
@ -15,60 +15,7 @@ from functools import partial
|
|||
import apis # from itself
|
||||
import traceback
|
||||
from item import Item # from itself
|
||||
|
||||
def link_this(src: str, dest: str):
|
||||
"""Creates symlink to source at destination
|
||||
If destination exists, it makes sure it is a (relative) symlink to source –
|
||||
if not, exception is being thrown"""
|
||||
|
||||
if not os.path.exists(src):
|
||||
raise Exception('Source file does not exist')
|
||||
# convert paths to relpaths
|
||||
src = os.path.relpath(src, os.getcwd())
|
||||
dest = os.path.relpath(dest, os.getcwd())
|
||||
|
||||
link_src = os.path.relpath(src, start=os.path.dirname(dest))
|
||||
if os.path.exists(dest):
|
||||
# check if existing destination is legitimate
|
||||
if not os.path.islink(dest):
|
||||
raise Exception('Destination file is not a symlink')
|
||||
if link_src != os.readlink(dest):
|
||||
raise Exception(f'Destination file points to different object')
|
||||
else:
|
||||
mkdir_this(os.path.dirname(dest))
|
||||
os.symlink(link_src, dest)
|
||||
|
||||
def mkdir_this(path: str):
|
||||
"""Creates directory and all intermediary directories in the path"""
|
||||
|
||||
path = os.path.normpath(path)
|
||||
to_create = []
|
||||
# find non-existing dirs
|
||||
while path and not os.path.exists(path):
|
||||
to_create.append(path)
|
||||
path = os.path.dirname(path)
|
||||
to_create.reverse()
|
||||
# create them
|
||||
for i in to_create:
|
||||
os.mkdir(i)
|
||||
|
||||
def scan_dir(path: str, orig_path=None) -> list:
|
||||
"""Iterates recursively through directory and returns list of all files.
|
||||
Does not follow symlinks
|
||||
"""
|
||||
|
||||
results = []
|
||||
if not orig_path: orig_path = path
|
||||
|
||||
for entry in os.scandir(path):
|
||||
# we don't want symlinks right now (smart scanning may be added in the future)
|
||||
if entry.is_symlink(): continue
|
||||
if entry.is_file():
|
||||
results.append(os.path.relpath(entry.path, start=orig_path))
|
||||
elif entry.is_dir():
|
||||
results += scan_dir(entry.path, orig_path)
|
||||
|
||||
return results
|
||||
from files import read_this, save_this, mkdir_this, link_this, scan_dir # from itself
|
||||
|
||||
def template_for(path: str) -> str:
|
||||
"""Determine used template for the file
|
||||
|
|
@ -76,19 +23,16 @@ def template_for(path: str) -> str:
|
|||
"""
|
||||
global config
|
||||
|
||||
template = None # the last matching will be used
|
||||
if 'templates' not in config:
|
||||
return None
|
||||
|
||||
template = template_data = None # the last matching will be used
|
||||
for pattern in config['templates'].keys():
|
||||
if fnmatch(path, pattern):
|
||||
template = config['templates'][pattern]
|
||||
return template
|
||||
|
||||
def save_this(path: str, content: str):
|
||||
"""Writes data to the file
|
||||
If the file or intermediary directories does not exist, create them
|
||||
"""
|
||||
mkdir_this(os.path.dirname(path))
|
||||
with open(path, 'w') as fp:
|
||||
fp.write(content)
|
||||
template_data = read_this('templates/'+template)
|
||||
|
||||
return template_data
|
||||
|
||||
def namespace_from(extension: type) -> dict:
|
||||
"""Generates global namespace for evaluation from provided class (extension)
|
||||
|
|
@ -106,7 +50,7 @@ def evaluate_this(content: str, global_ns: dict) -> str:
|
|||
|
||||
def evaluate(match: re.Match, global_ns: dict) -> str:
|
||||
match = match.group(1).split('\n')
|
||||
print('*','\n '.join(match))
|
||||
# print('*','\n '.join(match))
|
||||
# this trick let the functions in apis.py access our environment
|
||||
apis.environment = global_ns
|
||||
# now let's execute the command!
|
||||
|
|
@ -127,10 +71,11 @@ def evaluate(match: re.Match, global_ns: dict) -> str:
|
|||
def redirect(match: re.Match, domains: dict) -> str:
|
||||
this_domain = match.group(2)
|
||||
if this_domain in domains:
|
||||
print('>',f'{this_domain} => {domains[this_domain]}')
|
||||
# print('>',f'{this_domain} => {domains[this_domain]}')
|
||||
return match.group(1) + domains[this_domain] + match.group(3)
|
||||
else: return match.group(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
os.chdir(sys.path[0])
|
||||
|
||||
|
|
@ -157,9 +102,7 @@ if __name__ == '__main__':
|
|||
link_this('static/'+path, '../html/'+path)
|
||||
|
||||
|
||||
# now the actual parsing of content files
|
||||
template_cache = {} # not to load the same file multiple times
|
||||
|
||||
# now the actual parsing of content files:
|
||||
for path in content_files:
|
||||
print(path)
|
||||
|
||||
|
|
@ -173,20 +116,14 @@ if __name__ == '__main__':
|
|||
content = evaluate_this(item.content, namespace)
|
||||
|
||||
# load template
|
||||
if 'templates' in config:
|
||||
if template := template_for(path):
|
||||
if template not in template_cache:
|
||||
with open('templates/'+template) as fp:
|
||||
template_cache[template] = fp.read()
|
||||
template_data = template_cache[template]
|
||||
|
||||
# apply template
|
||||
namespace = namespace_from(apis.Content)
|
||||
namespace.update(config['variables'])
|
||||
namespace['path'] = item.path
|
||||
namespace.update(item.frontmatter_data)
|
||||
namespace['content'] = content
|
||||
content = evaluate_this(template_data, namespace)
|
||||
if template := template_for(path):
|
||||
# apply template
|
||||
namespace = namespace_from(apis.Content)
|
||||
namespace.update(config['variables'])
|
||||
namespace['path'] = item.path
|
||||
namespace.update(item.frontmatter_data)
|
||||
namespace['content'] = content
|
||||
content = evaluate_this(template, namespace)
|
||||
|
||||
# do redirections
|
||||
if 'redirections' in config:
|
||||
|
|
@ -200,8 +137,15 @@ if __name__ == '__main__':
|
|||
content = re.sub(r'=>(\s+)/(\S+)',
|
||||
lambda a: '=>'+a.group(1)+os.path.relpath(a.group(2), start=os.path.dirname(path)),
|
||||
content)
|
||||
|
||||
# TODO: produce HTML version
|
||||
|
||||
# produce HTML version
|
||||
html_path = os.path.splitext(path)[0]+'.html'
|
||||
# determine which template to use
|
||||
if template := template_for(html_path):
|
||||
...
|
||||
|
||||
# save results
|
||||
save_this('../'+path, content)
|
||||
|
||||
if not path.endswith('.gmi'):
|
||||
link_this('../'+path, '../html/'+path)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
import os
|
||||
|
||||
cache = {} # globally shared object to store file content
|
||||
|
||||
def read_this(path: str):
|
||||
"""Reads data from the file
|
||||
This function uses cache to minimize I/O
|
||||
"""
|
||||
global cache
|
||||
|
||||
if path not in cache:
|
||||
with open(path) as fp:
|
||||
cache[path] = fp.read()
|
||||
|
||||
return cache[path]
|
||||
|
||||
def save_this(path: str, content: str):
|
||||
"""Writes data to the file
|
||||
If the file or intermediary directories does not exist, create them
|
||||
This function uses cache to minimize I/O
|
||||
"""
|
||||
global cache
|
||||
|
||||
mkdir_this(os.path.dirname(path))
|
||||
with open(path, 'w') as fp:
|
||||
fp.write(content)
|
||||
cache[path] = content
|
||||
|
||||
def mkdir_this(path: str):
|
||||
"""Creates directory and all intermediary directories in the path"""
|
||||
|
||||
path = os.path.normpath(path)
|
||||
to_create = []
|
||||
# find non-existing dirs
|
||||
while path and not os.path.exists(path):
|
||||
to_create.append(path)
|
||||
path = os.path.dirname(path)
|
||||
to_create.reverse()
|
||||
# create them
|
||||
for i in to_create:
|
||||
os.mkdir(i)
|
||||
|
||||
def link_this(src: str, dest: str):
|
||||
"""Creates symlink to source at destination
|
||||
If destination exists, it makes sure it is a (relative) symlink to source –
|
||||
if not, exception is being thrown
|
||||
"""
|
||||
|
||||
if not os.path.exists(src):
|
||||
raise Exception('Source file does not exist')
|
||||
# convert paths to relpaths
|
||||
src = os.path.relpath(src, os.getcwd())
|
||||
dest = os.path.relpath(dest, os.getcwd())
|
||||
|
||||
link_src = os.path.relpath(src, start=os.path.dirname(dest))
|
||||
if os.path.exists(dest):
|
||||
# check if existing destination is legitimate
|
||||
if not os.path.islink(dest):
|
||||
raise Exception('Destination file is not a symlink')
|
||||
if link_src != os.readlink(dest):
|
||||
raise Exception(f'Destination file points to different object')
|
||||
else:
|
||||
mkdir_this(os.path.dirname(dest))
|
||||
os.symlink(link_src, dest)
|
||||
|
||||
def scan_dir(path: str, orig_path=None) -> list:
|
||||
"""Iterates recursively through directory and returns list of all files.
|
||||
Does not follow symlinks
|
||||
"""
|
||||
|
||||
results = []
|
||||
if not orig_path: orig_path = path
|
||||
|
||||
for entry in os.scandir(path):
|
||||
# we don't want symlinks right now (smart scanning may be added in the future)
|
||||
if entry.is_symlink(): continue
|
||||
if entry.is_file():
|
||||
results.append(os.path.relpath(entry.path, start=orig_path))
|
||||
elif entry.is_dir():
|
||||
results += scan_dir(entry.path, orig_path)
|
||||
|
||||
return results
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
import os
|
||||
import frontmatter # from package python-frontmatter
|
||||
import datetime
|
||||
from files import read_this # from itself
|
||||
|
||||
class Item:
|
||||
"""This class represents single content file
|
||||
It extracts some pre-defined frontmatter fields using python-frontmatter module"""
|
||||
|
||||
|
||||
title = None
|
||||
tags = []
|
||||
description = None
|
||||
|
|
@ -17,8 +18,7 @@ class Item:
|
|||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
with open('content/'+path) as fp:
|
||||
frontmatter_data, content = frontmatter.parse(fp.read())
|
||||
frontmatter_data, content = frontmatter.parse(read_this('content/'+path))
|
||||
self.content = content
|
||||
|
||||
if 'created' in frontmatter_data:
|
||||
|
|
|
|||
Reference in New Issue