trename files to proper module names; add docstrings - amprolla - devuan's apt repo merger
HTML git clone git://parazyd.org/amprolla.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit 1007576330d2f10a011e03651e24b696503b1299
DIR parent a95cb9c63b162446283a7b3a98e800a940c28ba4
HTML Author: parazyd <parazyd@dyne.org>
Date: Thu, 1 Jun 2017 01:16:32 +0200
rename files to proper module names; add docstrings
Diffstat:
D amprolla-download | 72 -------------------------------
D amprolla-merge.py | 118 -------------------------------
A amprolla_download.py | 76 +++++++++++++++++++++++++++++++
A amprolla_merge.py | 123 +++++++++++++++++++++++++++++++
M orchestrate.py | 2 +-
5 files changed, 200 insertions(+), 191 deletions(-)
---
DIR diff --git a/amprolla-download b/amprolla-download
t@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-# see LICENSE file for copyright and license details
-
-import subprocess
-from os.path import join
-import requests
-
-from lib.net import download
-from lib.parse import parse_release, get_time, get_date
-
-
-roots = {
- 'devuan': {
- 'local': 'spool/devuan/dists/jessie',
- 'remote': 'http://auto.mirror.devuan.org/devuan/dists/jessie',
- },
- 'debian': {
- 'local': 'spool/debian/dists/jessie',
- 'remote': 'http://ftp.debian.org/debian/dists/jessie',
- },
- 'debian-sec': {
- 'local': 'spool/dists/jessie/updates',
- 'remote': 'http://security.debian.org/dists/jessie/updates',
- },
-}
-
-release_file = 'Release'
-
-
-def merge_files(repo, relfile):
- """
- Loads the local release and call the merge process
- """
- print('Loading Release')
- rel = join(roots[repo]['local'], relfile)
- release_contents = open(rel).read()
-
- hashes = parse_release(release_contents)
-
- for k in hashes.keys():
- # if k.endswith('Packages.gz'):
- if k.endswith('/binary-armhf/Packages.gz'):
- # skip empty files
- # TODO: probably best to copy it in place when this occurs
- if hashes[k] == 'f61f27bd17de546264aa58f40f3aafaac7021e0ef69c17f6b1b4cd7664a037ec':
- print('Skipping %s' % k)
- continue
-
- subprocess.run(['./amprolla-merge', k])
-
-
-local_rel = join(roots['devuan']['local'], release_file)
-remote_rel = join(roots['devuan']['remote'], release_file)
-
-# Possibly use this var to check for changed hashes
-local_contents = open(local_rel).read()
-local_date = get_date(local_contents)
-
-r = requests.get(remote_rel)
-remote_contents = r.text
-remote_date = get_date(remote_contents)
-
-print('Local date: %s' % local_date)
-print('Remote date: %s' % remote_date)
-
-if get_time(remote_date) > get_time(local_date):
- # dump new release in place and merge
- # NOTE: when testing, watch out because you lose the old Release file in
- # spool
- print('Remote is newer')
- download(remote_rel, local_rel)
- merge_files('devuan', local_rel)
DIR diff --git a/amprolla-merge.py b/amprolla-merge.py
t@@ -1,118 +0,0 @@
-#!/usr/bin/env python3
-# see LICENSE file for copyright and license details
-
-"""
-Amprolla main module
-"""
-
-import sys
-from os.path import join
-from time import time
-
-from lib.package import (write_packages, load_packages_file,
- merge_packages_many)
-from lib.config import (aliases, banpkgs, repo_order, repos,
- spooldir, suites , mergedir, mergesubdir)
-
-
-def prepare_merge_dict():
- """
- This function will prepare a dict of lists that contain the repos
- that need to be merged in an ordered fashion. Orders them using the
- repo_order list found in lib.config
- Example output:
- {'ascii': ['ascii', None, 'stretch']},
- """
- merge_dict = {}
-
- for suite in suites:
- for i in suites[suite]:
- merge_dict[i] = []
-
- for suite in merge_dict:
- for repo in repo_order:
- tmpsuite = suite
- if repos[repo]['aliases'] is True:
- if tmpsuite in aliases[repos[repo]['name']]:
- tmpsuite = aliases[repos[repo]['name']][suite]
- elif repos[repo]['skipmissing'] is True:
- tmpsuite = None
- skips = ['jessie-security', 'ascii-security']
- if repo == 'debian' and suite in skips:
- tmpsuite = None
- if tmpsuite: # make it a proper path
- tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite)
- merge_dict[suite].append(tmpsuite)
-
- return merge_dict
-
-
-def devuan_rewrite(pkg, repo_name):
- """
- Function to be called when including a certain package. Allows for changing
- any attributes. Currently only changes the filename if we include a package
- when repo_name == 'devuan'.
- """
- if repo_name == 'devuan':
- pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' %
- repos[repo_name]['name'])
-
- return pkg
-
-
-def merge(packages_list):
- t1 = time()
-
- all_repos = []
- print('Loading packages: %s' % packages_list)
-
- devuan = load_packages_file(packages_list[0])
- if devuan:
- all_repos.append({'name': 'devuan', 'packages': devuan})
-
- debian_sec = load_packages_file(packages_list[1])
- if debian_sec:
- all_repos.append({'name': 'debian-sec', 'packages': debian_sec})
-
- debian = load_packages_file(packages_list[2])
- if debian:
- all_repos.append({'name': 'debian', 'packages': debian})
-
- print('Merging packages')
- new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, rewriter=devuan_rewrite)
-
- print('Writing packages')
- # replace the devuan subdir with our mergedir that we plan to fill
- new_out = packages_list[0].replace(join(spooldir,
- repos['devuan']['dists']),
- join(mergedir, mergesubdir))
- write_packages(new_pkgs, new_out)
-
- t2 = time()
- print('time:', t2-t1)
-
-
-def main(packages_file):
- #packages_file = 'main/binary-armhf/Packages.gz'
- #packages_file = argv[1]
- print(packages_file)
- to_merge = prepare_merge_dict()
-
- tt1 = time()
- for suite in to_merge:
- pkg_list = []
- for rep in to_merge[suite]:
- if rep:
- pkg_list.append(join(rep, packages_file))
- else:
- pkg_list.append(None)
-
- # print(pkg_list)
- merge(pkg_list)
-
- tt2 = time()
- print('total time:', tt2-tt1)
-
-
-if __name__ == '__main__':
- main(sys.argv[1])
DIR diff --git a/amprolla_download.py b/amprolla_download.py
t@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# see LICENSE file for copyright and license details
+
+"""
+module to pool and download Release files when needed
+"""
+
+import subprocess
+from os.path import join
+import requests
+
+from lib.net import download
+from lib.parse import parse_release, get_time, get_date
+
+
+roots = {
+ 'devuan': {
+ 'local': 'spool/devuan/dists/jessie',
+ 'remote': 'http://auto.mirror.devuan.org/devuan/dists/jessie',
+ },
+ 'debian': {
+ 'local': 'spool/debian/dists/jessie',
+ 'remote': 'http://ftp.debian.org/debian/dists/jessie',
+ },
+ 'debian-sec': {
+ 'local': 'spool/dists/jessie/updates',
+ 'remote': 'http://security.debian.org/dists/jessie/updates',
+ },
+}
+
+release_file = 'Release'
+
+
+def merge_files(repo, relfile):
+ """
+ Loads the local release and call the merge process
+ """
+ print('Loading Release')
+ rel = join(roots[repo]['local'], relfile)
+ release_contents = open(rel).read()
+
+ hashes = parse_release(release_contents)
+
+ for k in hashes:
+ # if k.endswith('Packages.gz'):
+ if k.endswith('/binary-armhf/Packages.gz'):
+ # skip empty files
+ # TODO: probably best to copy it in place when this occurs
+ if hashes[k] == 'f61f27bd17de546264aa58f40f3aafaac7021e0ef69c17f6b1b4cd7664a037ec':
+ print('Skipping %s' % k)
+ continue
+
+ subprocess.run(['./amprolla-merge', k])
+
+
+local_rel = join(roots['devuan']['local'], release_file)
+remote_rel = join(roots['devuan']['remote'], release_file)
+
+# Possibly use this var to check for changed hashes
+local_contents = open(local_rel).read()
+local_date = get_date(local_contents)
+
+r = requests.get(remote_rel)
+remote_contents = r.text
+remote_date = get_date(remote_contents)
+
+print('Local date: %s' % local_date)
+print('Remote date: %s' % remote_date)
+
+if get_time(remote_date) > get_time(local_date):
+ # dump new release in place and merge
+ # NOTE: when testing, watch out because you lose the old Release file in
+ # spool
+ print('Remote is newer')
+ download(remote_rel, local_rel)
+ merge_files('devuan', local_rel)
DIR diff --git a/amprolla_merge.py b/amprolla_merge.py
t@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# see LICENSE file for copyright and license details
+
+"""
+Amprolla main module
+"""
+
+from sys import argv
+from os.path import join
+from time import time
+
+from lib.package import (write_packages, load_packages_file,
+ merge_packages_many)
+from lib.config import (aliases, banpkgs, repo_order, repos,
+ spooldir, suites, mergedir, mergesubdir)
+
+
+def prepare_merge_dict():
+ """
+ This function will prepare a dict of lists that contain the repos
+ that need to be merged in an ordered fashion. Orders them using the
+ repo_order list found in lib.config
+ Example output:
+ {'ascii': ['ascii', None, 'stretch']},
+ """
+ merge_dict = {}
+
+ for suite in suites:
+ for i in suites[suite]:
+ merge_dict[i] = []
+
+ for suite in merge_dict:
+ for repo in repo_order:
+ tmpsuite = suite
+ if repos[repo]['aliases'] is True:
+ if tmpsuite in aliases[repos[repo]['name']]:
+ tmpsuite = aliases[repos[repo]['name']][suite]
+ elif repos[repo]['skipmissing'] is True:
+ tmpsuite = None
+ skips = ['jessie-security', 'ascii-security']
+ if repo == 'debian' and suite in skips:
+ tmpsuite = None
+ if tmpsuite: # make it a proper path
+ tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite)
+ merge_dict[suite].append(tmpsuite)
+
+ return merge_dict
+
+
+def devuan_rewrite(pkg, repo_name):
+ """
+ Function to be called when including a certain package. Allows for changing
+ any attributes. Currently only changes the filename if we include a package
+ when repo_name == 'devuan'.
+ """
+ if repo_name == 'devuan':
+ pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' %
+ repos[repo_name]['name'])
+
+ return pkg
+
+
+def merge(packages_list):
+ """
+ Merges the Packages files given in the package list
+ """
+ t1 = time()
+
+ all_repos = []
+ print('Loading packages: %s' % packages_list)
+
+ devuan = load_packages_file(packages_list[0])
+ if devuan:
+ all_repos.append({'name': 'devuan', 'packages': devuan})
+
+ debian_sec = load_packages_file(packages_list[1])
+ if debian_sec:
+ all_repos.append({'name': 'debian-sec', 'packages': debian_sec})
+
+ debian = load_packages_file(packages_list[2])
+ if debian:
+ all_repos.append({'name': 'debian', 'packages': debian})
+
+ print('Merging packages')
+ new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs,
+ rewriter=devuan_rewrite)
+
+ print('Writing packages')
+ # replace the devuan subdir with our mergedir that we plan to fill
+ new_out = packages_list[0].replace(join(spooldir,
+ repos['devuan']['dists']),
+ join(mergedir, mergesubdir))
+ write_packages(new_pkgs, new_out)
+
+ t2 = time()
+ print('time:', t2-t1)
+
+
+def main(packages_file):
+ """
+ Main function that calls the actual merge
+ """
+ # print(packages_file)
+ to_merge = prepare_merge_dict()
+
+ tt1 = time()
+ for suite in to_merge:
+ pkg_list = []
+ for rep in to_merge[suite]:
+ if rep:
+ pkg_list.append(join(rep, packages_file))
+ else:
+ pkg_list.append(None)
+
+ # print(pkg_list)
+ merge(pkg_list)
+
+ tt2 = time()
+ print('total time:', tt2-tt1)
+
+
+if __name__ == '__main__':
+ main(argv[1])
DIR diff --git a/orchestrate.py b/orchestrate.py
t@@ -14,7 +14,7 @@ for i in arches:
#print(pkg)
-am = __import__('amprolla-merge')
+am = __import__('amprolla_merge')
p = Pool(4)
p.map(am.main, pkg)