tparallelize Release generation - amprolla - devuan's apt repo merger
HTML git clone git://parazyd.org/amprolla.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit cf78750a6a7d40fe6025a81a2738f71a19a2a042
DIR parent 89497bf3ee81e2b724808fed0f409aa3c41c2444
HTML Author: parazyd <parazyd@dyne.org>
Date: Tue, 6 Jun 2017 20:25:16 +0200
parallelize Release generation
Diffstat:
M amprolla_merge.py | 62 +++++++++++++++++--------------
1 file changed, 34 insertions(+), 28 deletions(-)
---
DIR diff --git a/amprolla_merge.py b/amprolla_merge.py
t@@ -102,35 +102,33 @@ def merge(packages_list):
write_packages(new_pkgs, new_out)
-def gen_release(s):
+def gen_release(suite):
"""
- Generates a Release file for a given main suite (jessie/ascii/unstable)
+ Generates a Release file for a given suite (jessie/ascii/unstable)
"""
- for suite in suites[s]:
- filelist = []
- print('Crawling %s' % suite)
- rootdir = join(mergedir, mergesubdir, suite)
- for cat in categories:
- for arch in arches:
- if arch == 'source':
- flist = srcfiles
- else:
- flist = pkgfiles
+ filelist = []
+ print('Crawling %s' % suite)
+ rootdir = join(mergedir, mergesubdir, suite)
+ for cat in categories:
+ for arch in arches:
+ if arch == 'source':
+ flist = srcfiles
+ else:
+ flist = pkgfiles
- for fl in flist:
- filelist.append(join(rootdir, cat, arch, fl))
- if arch != 'source':
- filelist.append(join(rootdir, cat,
- 'debian-installer', arch, fl))
+ for i in flist:
+ filelist.append(join(rootdir, cat, arch, i))
+ if arch != 'source':
+ filelist.append(join(rootdir, cat,
+ 'debian-installer', arch, i))
- newrfl = join(rootdir, 'Release')
- oldrfl = newrfl.replace(join(mergedir, mergesubdir),
- join(spooldir, repos['devuan']['dists']))
+ newrfl = join(rootdir, 'Release')
+ oldrfl = newrfl.replace(join(mergedir, mergesubdir),
+ join(spooldir, repos['devuan']['dists']))
- print('Writing Release')
- write_release(oldrfl, newrfl, filelist, rootdir)
- # break
+ print('Writing Release')
+ write_release(oldrfl, newrfl, filelist, rootdir)
def main_merge(packages_file):
t@@ -167,11 +165,19 @@ def main():
pkg.append(join(j, i, mrgfile))
# pprint(pkg)
- p = Pool(4) # Set it to the number of CPUs you want to use
- p.map(main_merge, pkg)
-
- for st in suites:
- gen_release(st)
+ mrgpool = Pool(4) # Set it to the number of CPUs you want to use
+ mrgpool.map(main_merge, pkg)
+ mrgpool.close()
+
+ rel_list = []
+ for i in suites:
+ for j in suites[i]:
+ rel_list.append(j)
+ # gen_release(j)
+
+ relpool = Pool(4) # Set it to the number of CPUs you want to use
+ relpool.map(gen_release, rel_list)
+ relpool.close()
if __name__ == '__main__':