123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410 |
- #!/usr/bin/env python3
- import contextlib
- import logging
- import os
- import re
- import shutil
- import sys
- import subprocess
- from datetime import datetime, timedelta
- from io import BytesIO
- from threading import Lock, Timer
- from watchdog.events import FileSystemEventHandler
- from watchdog.observers import Observer
- from http import HTTPStatus
- from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
- CONFIG_FILE = 'serve_header.yml'
- MAKEFILE = 'Makefile'
- INCLUDE = 'include/nlohmann/'
- SINGLE_INCLUDE = 'single_include/nlohmann/'
- HEADER = 'json.hpp'
- DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
- JSON_VERSION_RE = re.compile(r'\s*#\s*define\s+NLOHMANN_JSON_VERSION_MAJOR\s+')
- class ExitHandler(logging.StreamHandler):
- def __init__(self, level):
- """."""
- super().__init__()
- self.level = level
- def emit(self, record):
- if record.levelno >= self.level:
- sys.exit(1)
- def is_project_root(test_dir='.'):
- makefile = os.path.join(test_dir, MAKEFILE)
- include = os.path.join(test_dir, INCLUDE)
- single_include = os.path.join(test_dir, SINGLE_INCLUDE)
- return (os.path.exists(makefile)
- and os.path.isfile(makefile)
- and os.path.exists(include)
- and os.path.exists(single_include))
- class DirectoryEventBucket:
- def __init__(self, callback, delay=1.2, threshold=0.8):
- """."""
- self.delay = delay
- self.threshold = timedelta(seconds=threshold)
- self.callback = callback
- self.event_dirs = set([])
- self.timer = None
- self.lock = Lock()
- def start_timer(self):
- if self.timer is None:
- self.timer = Timer(self.delay, self.process_dirs)
- self.timer.start()
- def process_dirs(self):
- result_dirs = []
- event_dirs = set([])
- with self.lock:
- self.timer = None
- while self.event_dirs:
- time, event_dir = self.event_dirs.pop()
- delta = datetime.now() - time
- if delta < self.threshold:
- event_dirs.add((time, event_dir))
- else:
- result_dirs.append(event_dir)
- self.event_dirs = event_dirs
- if result_dirs:
- self.callback(os.path.commonpath(result_dirs))
- if self.event_dirs:
- self.start_timer()
- def add_dir(self, path):
- with self.lock:
- # add path to the set of event_dirs if it is not a sibling of
- # a directory already in the set
- if not any(os.path.commonpath([path, event_dir]) == event_dir
- for (_, event_dir) in self.event_dirs):
- self.event_dirs.add((datetime.now(), path))
- if self.timer is None:
- self.start_timer()
- class WorkTree:
- make_command = 'make'
- def __init__(self, root_dir, tree_dir):
- """."""
- self.root_dir = root_dir
- self.tree_dir = tree_dir
- self.rel_dir = os.path.relpath(tree_dir, root_dir)
- self.name = os.path.basename(tree_dir)
- self.include_dir = os.path.abspath(os.path.join(tree_dir, INCLUDE))
- self.header = os.path.abspath(os.path.join(tree_dir, SINGLE_INCLUDE, HEADER))
- self.rel_header = os.path.relpath(self.header, root_dir)
- self.dirty = True
- self.build_count = 0
- t = os.path.getmtime(self.header)
- t = datetime.fromtimestamp(t)
- self.build_time = t.strftime(DATETIME_FORMAT)
- def __hash__(self):
- """."""
- return hash((self.tree_dir))
- def __eq__(self, other):
- """."""
- if not isinstance(other, type(self)):
- return NotImplemented
- return self.tree_dir == other.tree_dir
- def update_dirty(self, path):
- if self.dirty:
- return
- path = os.path.abspath(path)
- if os.path.commonpath([path, self.include_dir]) == self.include_dir:
- logging.info(f'{self.name}: working tree marked dirty')
- self.dirty = True
- def amalgamate_header(self):
- if not self.dirty:
- return
- mtime = os.path.getmtime(self.header)
- subprocess.run([WorkTree.make_command, 'amalgamate'], cwd=self.tree_dir,
- stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
- if mtime == os.path.getmtime(self.header):
- logging.info(f'{self.name}: no changes')
- else:
- self.build_count += 1
- self.build_time = datetime.now().strftime(DATETIME_FORMAT)
- logging.info(f'{self.name}: header amalgamated (build count {self.build_count})')
- self.dirty = False
- class WorkTrees(FileSystemEventHandler):
- def __init__(self, root_dir):
- """."""
- super().__init__()
- self.root_dir = root_dir
- self.trees = set([])
- self.tree_lock = Lock()
- self.scan(root_dir)
- self.created_bucket = DirectoryEventBucket(self.scan)
- self.observer = Observer()
- self.observer.schedule(self, root_dir, recursive=True)
- self.observer.start()
- def scan(self, base_dir):
- scan_dirs = set([base_dir])
- # recursively scan base_dir for working trees
- while scan_dirs:
- scan_dir = os.path.abspath(scan_dirs.pop())
- self.scan_tree(scan_dir)
- try:
- with os.scandir(scan_dir) as dir_it:
- for entry in dir_it:
- if entry.is_dir():
- scan_dirs.add(entry.path)
- except FileNotFoundError as e:
- logging.debug('path disappeared: %s', e)
- def scan_tree(self, scan_dir):
- if not is_project_root(scan_dir):
- return
- # skip source trees in build directories
- # this check could be enhanced
- if scan_dir.endswith('/_deps/json-src'):
- return
- tree = WorkTree(self.root_dir, scan_dir)
- with self.tree_lock:
- if not tree in self.trees:
- if tree.name == tree.rel_dir:
- logging.info(f'adding working tree {tree.name}')
- else:
- logging.info(f'adding working tree {tree.name} at {tree.rel_dir}')
- url = os.path.join('/', tree.rel_dir, HEADER)
- logging.info(f'{tree.name}: serving header at {url}')
- self.trees.add(tree)
- def rescan(self, path=None):
- if path is not None:
- path = os.path.abspath(path)
- trees = set([])
- # check if any working trees have been removed
- with self.tree_lock:
- while self.trees:
- tree = self.trees.pop()
- if ((path is None
- or os.path.commonpath([path, tree.tree_dir]) == tree.tree_dir)
- and not is_project_root(tree.tree_dir)):
- if tree.name == tree.rel_dir:
- logging.info(f'removing working tree {tree.name}')
- else:
- logging.info(f'removing working tree {tree.name} at {tree.rel_dir}')
- else:
- trees.add(tree)
- self.trees = trees
- def find(self, path):
- # find working tree for a given header file path
- path = os.path.abspath(path)
- with self.tree_lock:
- for tree in self.trees:
- if path == tree.header:
- return tree
- return None
- def on_any_event(self, event):
- logging.debug('%s (is_dir=%s): %s', event.event_type,
- event.is_directory, event.src_path)
- path = os.path.abspath(event.src_path)
- if event.is_directory:
- if event.event_type == 'created':
- # check for new working trees
- self.created_bucket.add_dir(path)
- elif event.event_type == 'deleted':
- # check for deleted working trees
- self.rescan(path)
- elif event.event_type == 'closed':
- with self.tree_lock:
- for tree in self.trees:
- tree.update_dirty(path)
- def stop(self):
- self.observer.stop()
- self.observer.join()
- class HeaderRequestHandler(SimpleHTTPRequestHandler): # lgtm[py/missing-call-to-init]
- def __init__(self, request, client_address, server):
- """."""
- self.worktrees = server.worktrees
- self.worktree = None
- try:
- super().__init__(request, client_address, server,
- directory=server.worktrees.root_dir)
- except ConnectionResetError:
- logging.debug('connection reset by peer')
- def translate_path(self, path):
- path = os.path.abspath(super().translate_path(path))
- # add single_include/nlohmann into path, if needed
- header = os.path.join('/', HEADER)
- header_path = os.path.join('/', SINGLE_INCLUDE, HEADER)
- if (path.endswith(header)
- and not path.endswith(header_path)):
- path = os.path.join(os.path.dirname(path), SINGLE_INCLUDE, HEADER)
- return path
- def send_head(self):
- # check if the translated path matches a working tree
- # and fullfill the request; otherwise, send 404
- path = self.translate_path(self.path)
- self.worktree = self.worktrees.find(path)
- if self.worktree is not None:
- self.worktree.amalgamate_header()
- logging.info(f'{self.worktree.name}; serving header (build count {self.worktree.build_count})')
- return super().send_head()
- logging.info(f'invalid request path: {self.path}')
- super().send_error(HTTPStatus.NOT_FOUND, 'Not Found')
- return None
- def send_header(self, keyword, value):
- # intercept Content-Length header; sent in copyfile later
- if keyword == 'Content-Length':
- return
- super().send_header(keyword, value)
- def end_headers (self):
- # intercept; called in copyfile() or indirectly
- # by send_head via super().send_error()
- pass
- def copyfile(self, source, outputfile):
- injected = False
- content = BytesIO()
- length = 0
- # inject build count and time into served header
- for line in source:
- line = line.decode('utf-8')
- if not injected and JSON_VERSION_RE.match(line):
- length += content.write(bytes('#define JSON_BUILD_COUNT '\
- f'{self.worktree.build_count}\n', 'utf-8'))
- length += content.write(bytes('#define JSON_BUILD_TIME '\
- f'"{self.worktree.build_time}"\n\n', 'utf-8'))
- injected = True
- length += content.write(bytes(line, 'utf-8'))
- # set content length
- super().send_header('Content-Length', length)
- # CORS header
- self.send_header('Access-Control-Allow-Origin', '*')
- # prevent caching
- self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
- self.send_header('Pragma', 'no-cache')
- self.send_header('Expires', '0')
- super().end_headers()
- # send the header
- content.seek(0)
- shutil.copyfileobj(content, outputfile)
- def log_message(self, format, *args):
- pass
- class DualStackServer(ThreadingHTTPServer):
- def __init__(self, addr, worktrees):
- """."""
- self.worktrees = worktrees
- super().__init__(addr, HeaderRequestHandler)
- def server_bind(self):
- # suppress exception when protocol is IPv4
- with contextlib.suppress(Exception):
- self.socket.setsockopt(
- socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
- return super().server_bind()
- if __name__ == '__main__':
- import argparse
- import ssl
- import socket
- import yaml
- # exit code
- ec = 0
- # setup logging
- logging.basicConfig(format='[%(asctime)s] %(levelname)s: %(message)s',
- datefmt=DATETIME_FORMAT, level=logging.INFO)
- log = logging.getLogger()
- log.addHandler(ExitHandler(logging.ERROR))
- # parse command line arguments
- parser = argparse.ArgumentParser()
- parser.add_argument('--make', default='make',
- help='the make command (default: make)')
- args = parser.parse_args()
- # propagate the make command to use for amalgamating headers
- WorkTree.make_command = args.make
- worktrees = None
- try:
- # change working directory to project root
- os.chdir(os.path.realpath(os.path.join(sys.path[0], '../../')))
- if not is_project_root():
- log.error('working directory does not look like project root')
- # load config
- config = {}
- config_file = os.path.abspath(CONFIG_FILE)
- try:
- with open(config_file, 'r') as f:
- config = yaml.safe_load(f)
- except FileNotFoundError:
- log.info(f'cannot find configuration file: {config_file}')
- log.info('using default configuration')
- # find and monitor working trees
- worktrees = WorkTrees(config.get('root', '.'))
- # start web server
- infos = socket.getaddrinfo(config.get('bind', None), config.get('port', 8443),
- type=socket.SOCK_STREAM, flags=socket.AI_PASSIVE)
- DualStackServer.address_family = infos[0][0]
- HeaderRequestHandler.protocol_version = 'HTTP/1.0'
- with DualStackServer(infos[0][4], worktrees) as httpd:
- scheme = 'HTTP'
- https = config.get('https', {})
- if https.get('enabled', True):
- cert_file = https.get('cert_file', 'localhost.pem')
- key_file = https.get('key_file', 'localhost-key.pem')
- ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
- ssl_ctx.minimum_version = ssl.TLSVersion.TLSv1_2
- ssl_ctx.maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED
- ssl_ctx.load_cert_chain(cert_file, key_file)
- httpd.socket = ssl_ctx.wrap_socket(httpd.socket, server_side=True)
- scheme = 'HTTPS'
- host, port = httpd.socket.getsockname()[:2]
- log.info(f'serving {scheme} on {host} port {port}')
- log.info('press Ctrl+C to exit')
- httpd.serve_forever()
- except KeyboardInterrupt:
- log.info('exiting')
- except Exception:
- ec = 1
- log.exception('an error occurred:')
- finally:
- if worktrees is not None:
- worktrees.stop()
- sys.exit(ec)
|