+"""Includes classes that read the configuration and traverse the file system to
+find matching objects to use to freeze the distribution.
+
+"""
+__author__ = 'Paul Landes'
+
+from typing import Dict, Any
+import os
+import stat
+import socket
+import logging
+import json
+import zipfile
+from pathlib import Path
+from datetime import datetime
+from zensols.persist import persisted
+from zensols.grsync import (
+ RepoSpec,
+ SymbolicLink,
+ BootstrapGenerator,
+ PathTranslator,
+ AppConfig,
+)
+
+logger = logging.getLogger(__name__)
+
+
+
+
[docs]
+
class Discoverer(object):
+
"""Discover git repositories, links, files and directories to save to
+
reconstitute a user home directory later.
+
+
"""
+
CONF_TARG_KEY = 'discover.target.config'
+
TARG_LINKS = 'discover.target.links'
+
REPO_PREF = 'discover.repo.remote_pref'
+
+
+
[docs]
+
def __init__(self, config: AppConfig, profiles: list,
+
path_translator: PathTranslator, repo_preference: str):
+
self.config = config
+
self.profiles_override = profiles
+
self.path_translator = path_translator
+
self._repo_preference = repo_preference
+
+
+
def _get_repo_paths(self, paths):
+
"""Recusively find git repository root directories."""
+
git_paths = []
+
logger.debug('repo root search paths {}'.format(paths))
+
for path in paths:
+
logger.debug('searching git paths in {}'.format(path.resolve()))
+
for root, dirs, files in os.walk(path.resolve()):
+
rootpath = Path(root)
+
if rootpath.name == '.git':
+
git_paths.append(rootpath.parent)
+
return git_paths
+
+
def _discover_repo_specs(self, paths, links):
+
"""Return a list of RepoSpec objects.
+
+
:param paths: a list of paths each of which start a new RepoSpec
+
:param links: a list of symlinks to check if they point to the
+
repository, and if so, add them to the RepoSpec
+
+
"""
+
repo_specs = []
+
logger.debug(f'repo spec paths: {paths}')
+
for path in paths:
+
logger.debug(f'found repo at path {path}')
+
repo_spec = RepoSpec(path, self.path_translator)
+
repo_spec.add_linked(links)
+
if len(repo_spec.remotes) == 0:
+
logger.warning(f'repo {repo_spec} has no remotes--skipping...')
+
else:
+
repo_specs.append(repo_spec)
+
return repo_specs
+
+
@property
+
@persisted('_profiles')
+
def profiles(self):
+
if self.config is None:
+
raise ValueError('no configuration given; use the --help option')
+
return self.config.get_profiles(self.profiles_override)
+
+
+
[docs]
+
def get_discoverable_objects(self):
+
"""Find git repos, files, sym links and directories to reconstitute
+
later.
+
+
"""
+
paths = []
+
if logger.isEnabledFor(logging.INFO):
+
path: Path = {self.config.config_file}
+
logger.info(f'finding objects to perist defined in {path}')
+
for fname in self.config.get_discoverable_objects(self.profiles):
+
path = Path(fname).expanduser().absolute()
+
logger.debug(f'file pattern {fname} -> {path}')
+
bname = path.name
+
dname = path.parent.expanduser()
+
files = list(dname.glob(bname))
+
logger.debug(f'expanding {path} -> {dname} / {bname}: {files}')
+
paths.extend(files)
+
return paths
+
+
+
def _create_file(self, src, dst=None, no_path_obj=False, robust=False):
+
"""Return a file object, which has the relative (rel) to home dir path,
+
absolute path (abs) used later to zip the file, and mode (mode and
+
modestr) information.
+
+
"""
+
dst = src if dst is None else dst
+
if src.exists():
+
mode = src.stat().st_mode
+
modestr = stat.filemode(mode)
+
modify_time = os.path.getmtime(src)
+
create_time = os.path.getctime(src)
+
elif not robust:
+
raise OSError(f'no such file: {src}')
+
else:
+
logger.warning(f'missing file: {src}--robustly skipping')
+
mode, modestr, create_time, modify_time = None, None, None, None
+
# the mode string is used as documentation and currently there is no
+
# way to convert from a mode string to an octal mode, which would be
+
# nice to allow modification of the dist.json file.
+
fobj = {'modestr': modestr,
+
'mode': mode,
+
'create_time': create_time,
+
'modify_time': modify_time}
+
if no_path_obj:
+
fobj['rel'] = str(self.path_translator.relative_to(dst))
+
else:
+
fobj['abs'] = src
+
fobj['rel'] = self.path_translator.relative_to(dst)
+
return fobj
+
+
+
[docs]
+
def discover(self, flatten) -> Dict[str, Any]:
+
"""Main worker method to capture all the user home information (git repos,
+
files, sym links and empty directories per the configuration file).
+
+
:param flatten: if ``True`` then return a data structure appropriate
+
for pretty printing; this will omit data needed to
+
create the distrubtion so it shouldn't be used for the
+
freeze task
+
+
"""
+
files = []
+
dirs = []
+
empty_dirs = []
+
pattern_links = []
+
path_trans = self.path_translator
+
+
# find all things to persist (repos, symlinks, files, etc)
+
dobjs = self.get_discoverable_objects()
+
# all directories are either repositories or base directories to
+
# persist files in the distribution file
+
dirs_or_gits = tuple(
+
filter(lambda x: x.is_dir() and not x.is_symlink(), dobjs))
+
# find the directories that have git repos in them (recursively)
+
git_paths = self._get_repo_paths(dirs_or_gits)
+
# create symbolic link objects from those objects that are links
+
links = tuple(map(lambda l: SymbolicLink(l, self.path_translator),
+
filter(lambda x: x.is_symlink(), dobjs)))
+
#logger.debug(f'links: {links}')
+
# normal files are kept track of so we can compress them later
+
for f in filter(lambda x: x.is_file() and not x.is_symlink(), dobjs):
+
files.append(self._create_file(f))
+
# create RepoSpec objects that capture information about the git repo
+
repo_specs = self._discover_repo_specs(git_paths, links)
+
# these are the Path objects to where the repo lives on the local fs
+
repo_paths = set(map(lambda x: x.path, repo_specs))
+
# add the configuration used to freeze so the target can freeze again
+
if self.config.has_option(self.CONF_TARG_KEY):
+
config_targ = self.config.get_option(self.CONF_TARG_KEY)
+
src = Path(self.config.config_file)
+
dst = Path(config_targ).expanduser()
+
files.append(self._create_file(dst, dst))
+
logger.debug(f'files: {files}')
+
+
# recusively find files that don't belong to git repos
+
def gather(par):
+
for c in par.iterdir():
+
if c.is_dir() and c not in repo_paths:
+
gather(c)
+
elif c.is_file():
+
files.append(self._create_file(c))
+
+
# find files that don't belong to git repos
+
for path in filter(lambda x: x not in repo_paths, dirs_or_gits):
+
logger.debug('dir {}'.format(path))
+
dirs.append({'abs': path, 'rel': path_trans.relative_to(path)})
+
gather(path)
+
+
# configurated empty directories are added only if they exist so we can
+
# recreate with the correct mode
+
logger.info(f'using profiles: {", ".join(self.profiles)}')
+
for ed in self.config.get_empty_dirs(self.profiles):
+
logger.debug('empty dir: {}'.format(str(ed)))
+
empty_dirs.append(self._create_file(
+
ed, no_path_obj=True, robust=True))
+
+
# pattern symlinks are special links that can change name based on
+
# variables like the platform name so each link points to a
+
# configuration file for that platform.
+
if self.config.has_option(self.TARG_LINKS):
+
dec_links = self.config.get_option(self.TARG_LINKS)
+
for link in map(lambda x: x['link'],
+
filter(lambda x: 'link' in x, dec_links)):
+
src = Path(link['source']).expanduser().absolute()
+
targ = Path(link['target']).expanduser().absolute()
+
pattern_links.append(
+
{'source': str(path_trans.relative_to(src)),
+
'target': str(path_trans.relative_to(targ))})
+
+
# create data structures for symbolic link integrity
+
files_by_name = {f['abs']: f for f in files}
+
for f in files:
+
if f['abs'].is_file():
+
dname = f['abs'].parent
+
files_by_name[dname] = dname
+
if flatten:
+
del f['abs']
+
f['rel'] = str(f['rel'])
+
+
# unused links pointing to repositories won't get created, so those not
+
# used by repos are added explicitly to pattern links
+
for link in links:
+
if link.use_count == 0:
+
try:
+
pattern_links.append(
+
{'source': str(link.source_relative),
+
'target': str(link.target_relative)})
+
except ValueError as e:
+
logger.error(f'couldn\'t create link: {link}')
+
raise e
+
if link.target in files_by_name:
+
dst = files_by_name[link.target]
+
# follow links enhancement picks up here
+
logger.debug(f'source {link.source} -> {dst}')
+
else:
+
logger.warning(f'hanging link with no target: {link}--' +
+
'proceeding anyway')
+
+
return {'repo_specs': repo_specs,
+
'empty_dirs': empty_dirs,
+
'files': files,
+
'links': pattern_links}
+
+
+
@property
+
def repo_preference(self):
+
"""Return the preference for which repo to make primary on thaw
+
+
"""
+
return self._repo_preference or \
+
(self.config.has_option(self.REPO_PREF) and
+
self.config.get_option(self.REPO_PREF))
+
+
+
[docs]
+
def freeze(self, flatten=False):
+
"""Main entry point method that creates an object graph of all the data that
+
needs to be saved (freeze) in the user home directory to reconstitute
+
later (thaw).
+
+
:param flatten: if ``True`` then return a data structure appropriate
+
for pretty printing; this will omit data needed to
+
create the distrubtion so it shouldn't be used for the
+
freeze task
+
+
"""
+
disc = self.discover(flatten)
+
repo_specs = tuple(x.freeze() for x in disc['repo_specs'])
+
files = disc['files']
+
logger.info('freeezing with git repository ' +
+
f'preference: {self.repo_preference}')
+
disc.update({'repo_specs': repo_specs,
+
'repo_pref': self.repo_preference,
+
'files': files,
+
'source': socket.gethostname(),
+
'create_date': datetime.now().isoformat(
+
timespec='minutes')})
+
return disc
+
+
[docs]
+
class FreezeManager(object):
+
"""Invoked by a client to create *frozen* distribution .
+
+
"""
+
CREATE_WHEEL = 'discover.wheel.create'
+
+
+
[docs]
+
def __init__(self, config, dist_file, defs_file, discoverer, app_version,
+
dry_run: bool):
+
self.config = config
+
self.dist_file = dist_file
+
self.defs_file = defs_file
+
self.discoverer = discoverer
+
self.app_version = app_version
+
self.dry_run = dry_run
+
+
+
def _create_wheels(self, wheel_dependency):
+
"""Create wheel dependencies on this software so the host doesn't need Internet
+
connectivity. Currently the YAML dependency breaks this since only
+
binary per host wheels are available for download and the wrong was is
+
given of spanning platforms (i.e. OSX to Linux).
+
+
"""
+
wheel_dir_name = self.config.wheel_dir_name
+
wheel_dir = Path(self.dist_dir, wheel_dir_name)
+
logger.info(f'creating wheels from dependency {wheel_dependency} in {wheel_dir}')
+
if not wheel_dir.exists():
+
wheel_dir.mkdir(parents=True, exist_ok=True)
+
from pip._internal import main
+
pip_cmd = f'wheel --wheel-dir={wheel_dir} --no-cache-dir {wheel_dependency}'
+
logger.debug('pip cmd: {}'.format(pip_cmd))
+
main(pip_cmd.split())
+
+
def _freeze_dist(self):
+
"""Freeze the distribution (see the class documentation).
+
+
"""
+
dist_dir = self.dist_file.parent
+
if not self.dry_run and not dist_dir.exists():
+
dist_dir.mkdir(parents=True, exist_ok=True)
+
data = self.discoverer.freeze()
+
data['app_version'] = self.app_version
+
if not self.dry_run:
+
with zipfile.ZipFile(self.dist_file, mode='w') as zf:
+
for finfo in data['files']:
+
fabs = finfo['abs']
+
frel = str(Path(finfo['rel']))
+
if logger.isEnabledFor(logging.DEBUG):
+
logger.debug(f'adding file: {fabs}')
+
zf.write(fabs, arcname=frel)
+
del finfo['abs']
+
finfo['rel'] = frel
+
logger.info(f'writing distribution defs to {self.defs_file}')
+
zf.writestr(self.defs_file, json.dumps(data, indent=2))
+
logger.info(f'created frozen distribution in {self.dist_file}')
+
+
+
[docs]
+
def freeze(self, wheel_dependency=None):
+
"""Freeze the distribution by saving creating a script to thaw along with all
+
artifacts (i.e. repo definitions) in a zip file.
+
+
"""
+
self._freeze_dist()
+
script_file = self.config.bootstrap_script_file
+
if not self.dry_run:
+
bg = BootstrapGenerator(self.config)
+
bg.generate(script_file)
+
script_file.chmod(0o755)
+
# wheel creation last since pip clobers/reconfigures logging
+
if self.config.has_option(self.CREATE_WHEEL):
+
create_wheel = self.config.get_option(self.CREATE_WHEEL)
+
if create_wheel and wheel_dependency is not None:
+
self._create_wheels(wheel_dependency)
+