tadd a orchestrate merge example - amprolla - devuan's apt repo merger
HTML git clone git://parazyd.org/amprolla.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit 7b234b5febe9b32148c60a7ff5b5f582e5882d58
DIR parent 385ad0a13cbf41188892e9880db84353a19657b6
HTML Author: parazyd <parazyd@dyne.org>
Date: Wed, 31 May 2017 23:07:42 +0200
add a orchestrate merge example
Diffstat:
D amprolla-merge | 107 -------------------------------
A amprolla-merge.py | 116 ++++++++++++++++++++++++++++++
M lib/config.py | 2 ++
A orchestrate.py | 19 +++++++++++++++++++
4 files changed, 137 insertions(+), 107 deletions(-)
---
DIR diff --git a/amprolla-merge b/amprolla-merge
t@@ -1,107 +0,0 @@
-#!/usr/bin/env python3
-"""
-Amprolla main module
-"""
-
-from os.path import join
-from time import time
-
-from lib.package import (write_packages, load_packages_file,
- merge_packages_many)
-from lib.config import (aliases, banpkgs, repo_order, repos,
- spooldir, suites , mergedir, mergesubdir)
-
-
-def prepare_merge_dict():
- """
- This function will prepare a dict of lists that contain the repos
- that need to be merged in an ordered fashion. Orders them using the
- repo_order list found in lib.config
- Example output:
- {'ascii': ['ascii', None, 'stretch']},
- """
- merge_dict = {}
-
- for suite in suites:
- for i in suites[suite]:
- merge_dict[i] = []
-
- for suite in merge_dict:
- for repo in repo_order:
- tmpsuite = suite
- if repos[repo]['aliases'] is True:
- if tmpsuite in aliases[repos[repo]['name']]:
- tmpsuite = aliases[repos[repo]['name']][suite]
- elif repos[repo]['skipmissing'] is True:
- tmpsuite = None
- skips = ['jessie-security', 'ascii-security']
- if repo == 'debian' and suite in skips:
- tmpsuite = None
- if tmpsuite: # make it a proper path
- tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite)
- merge_dict[suite].append(tmpsuite)
-
- return merge_dict
-
-
-def devuan_rewrite(pkg, repo_name):
- """
- Function to be called when including a certain package. Allows for changing
- any attributes. Currently only changes the filename if we include a package
- when repo_name == 'devuan'.
- """
- if repo_name == 'devuan':
- pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' %
- repos[repo_name]['name'])
-
- return pkg
-
-
-def merge(packages_list):
- t1 = time()
-
- all_repos = []
- print('Loading packages: %s' % packages_list)
-
- devuan = load_packages_file(packages_list[0])
- if devuan:
- all_repos.append({'name': 'devuan', 'packages': devuan})
-
- debian_sec = load_packages_file(packages_list[1])
- if debian_sec:
- all_repos.append({'name': 'debian-sec', 'packages': debian_sec})
-
- debian = load_packages_file(packages_list[2])
- if debian:
- all_repos.append({'name': 'debian', 'packages': debian})
-
- print('Merging packages')
- new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, rewriter=devuan_rewrite)
-
- print('Writing packages')
- # replace the devuan subdir with our mergedir that we plan to fill
- new_out = packages_list[0].replace(join(spooldir,
- repos['devuan']['dists']),
- join(mergedir, mergesubdir))
- write_packages(new_pkgs, new_out)
-
- t2 = time()
- print('time:', t2-t1)
-
-
-packages_file = 'main/binary-armhf/Packages.gz'
-to_merge = prepare_merge_dict()
-
-tt1 = time()
-for suite in to_merge:
- pkg_list = []
- for rep in to_merge[suite]:
- if rep:
- pkg_list.append(join(rep, packages_file))
- else:
- pkg_list.append(None)
-
- merge(pkg_list)
-
-tt2 = time()
-print('total time:', tt2-tt1)
DIR diff --git a/amprolla-merge.py b/amprolla-merge.py
t@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Amprolla main module
+"""
+
+import sys
+from os.path import join
+from time import time
+
+from lib.package import (write_packages, load_packages_file,
+ merge_packages_many)
+from lib.config import (aliases, banpkgs, repo_order, repos,
+ spooldir, suites , mergedir, mergesubdir)
+
+
+def prepare_merge_dict():
+ """
+ This function will prepare a dict of lists that contain the repos
+ that need to be merged in an ordered fashion. Orders them using the
+ repo_order list found in lib.config
+ Example output:
+ {'ascii': ['ascii', None, 'stretch']},
+ """
+ merge_dict = {}
+
+ for suite in suites:
+ for i in suites[suite]:
+ merge_dict[i] = []
+
+ for suite in merge_dict:
+ for repo in repo_order:
+ tmpsuite = suite
+ if repos[repo]['aliases'] is True:
+ if tmpsuite in aliases[repos[repo]['name']]:
+ tmpsuite = aliases[repos[repo]['name']][suite]
+ elif repos[repo]['skipmissing'] is True:
+ tmpsuite = None
+ skips = ['jessie-security', 'ascii-security']
+ if repo == 'debian' and suite in skips:
+ tmpsuite = None
+ if tmpsuite: # make it a proper path
+ tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite)
+ merge_dict[suite].append(tmpsuite)
+
+ return merge_dict
+
+
+def devuan_rewrite(pkg, repo_name):
+ """
+ Function to be called when including a certain package. Allows for changing
+ any attributes. Currently only changes the filename if we include a package
+ when repo_name == 'devuan'.
+ """
+ if repo_name == 'devuan':
+ pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' %
+ repos[repo_name]['name'])
+
+ return pkg
+
+
+def merge(packages_list):
+ t1 = time()
+
+ all_repos = []
+ print('Loading packages: %s' % packages_list)
+
+ devuan = load_packages_file(packages_list[0])
+ if devuan:
+ all_repos.append({'name': 'devuan', 'packages': devuan})
+
+ debian_sec = load_packages_file(packages_list[1])
+ if debian_sec:
+ all_repos.append({'name': 'debian-sec', 'packages': debian_sec})
+
+ debian = load_packages_file(packages_list[2])
+ if debian:
+ all_repos.append({'name': 'debian', 'packages': debian})
+
+ print('Merging packages')
+ new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, rewriter=devuan_rewrite)
+
+ print('Writing packages')
+ # replace the devuan subdir with our mergedir that we plan to fill
+ new_out = packages_list[0].replace(join(spooldir,
+ repos['devuan']['dists']),
+ join(mergedir, mergesubdir))
+ write_packages(new_pkgs, new_out)
+
+ t2 = time()
+ print('time:', t2-t1)
+
+
+def main(packages_file):
+ #packages_file = 'main/binary-armhf/Packages.gz'
+ #packages_file = argv[1]
+ print(packages_file)
+ to_merge = prepare_merge_dict()
+
+ tt1 = time()
+ for suite in to_merge:
+ pkg_list = []
+ for rep in to_merge[suite]:
+ if rep:
+ pkg_list.append(join(rep, packages_file))
+ else:
+ pkg_list.append(None)
+
+ # print(pkg_list)
+ merge(pkg_list)
+
+ tt2 = time()
+ print('total time:', tt2-tt1)
+
+
+if __name__ == '__main__':
+ main(sys.argv[1])
DIR diff --git a/lib/config.py b/lib/config.py
t@@ -73,6 +73,8 @@ aliases = {
categories = ['main', 'contrib', 'non-free']
+arches = ['binary-armhf']
+
mainrepofiles = [
'InRelease',
'Release',
DIR diff --git a/orchestrate.py b/orchestrate.py
t@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+from os.path import join
+from lib.config import arches, categories
+from multiprocessing import Pool
+
+
+pkg = []
+for i in arches:
+ for j in categories:
+ pkg.append(join(j, i, 'Packages.gz'))
+ pkg.append(join(j, 'debian-installer', i, 'Packages.gz'))
+
+
+#print(pkg)
+am = __import__('amprolla-merge')
+
+p = Pool(4)
+p.map(am.main, pkg)