tpylint/pep8 fixes, remove some cruft from lib - amprolla - devuan's apt repo merger
HTML git clone git://parazyd.org/amprolla.git
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit f9d091ad6f95301ec3909b87cf1d60930fea702f
DIR parent dcaa731f87ed924751ac5e25c5d6cc4a00f39e59
HTML Author: parazyd <parazyd@dyne.org>
Date: Thu, 1 Jun 2017 01:08:49 +0200
pylint/pep8 fixes, remove some cruft from lib
Diffstat:
M lib/config.py | 31 ++++++++++++++++++++++++++++---
D lib/fs.py | 34 -------------------------------
M lib/log.py | 7 ++++---
M lib/net.py | 17 +++++++++++------
M lib/package.py | 6 ++++++
M lib/parse.py | 78 +++++++++++--------------------
6 files changed, 77 insertions(+), 96 deletions(-)
---
DIR diff --git a/lib/config.py b/lib/config.py
t@@ -1,12 +1,17 @@
-#!/usr/bin/env python
-# copyright (c) 2017 - Ivan J. <parazyd@dyne.org>
# see LICENSE file for copyright and license details
+"""
+amprolla configuration file
+"""
+
+# from hashlib import md5, sha1, sha256
+
spooldir = './spool'
sign_key = 'fa1b0274'
mergedir = './merged'
mergesubdir = 'dists'
banpkgs = {'systemd', 'systemd-sysv'}
+# checksums = [{'name': 'MD5Sum', 'f': md5}, sha1, sha256]
# checksums = [ 'md5sum', 'sha1', 'sha256', 'sha512' ]
repo_order = ['devuan', 'debian-sec', 'debian']
t@@ -73,7 +78,27 @@ aliases = {
categories = ['main', 'contrib', 'non-free']
-arches = ['binary-armhf']
+arches = [
+ # 'binary-all',
+ # 'binary-alpha',
+ # 'binary-amd64',
+ # 'binary-arm64',
+ # 'binary-armel',
+ 'binary-armhf',
+ # 'binary-hppa',
+ # 'binary-hurd-i386',
+ # 'binary-i386',
+ # 'binary-ia64',
+ # 'binary-kfreebsd-amd64',
+ # 'binary-kfreebsd-i386',
+ # 'binary-mips',
+ # 'binary-mips64el',
+ # 'binary-mipsel',
+ # 'binary-powerpc',
+ # 'binary-ppc64el',
+ # 'binary-s390x',
+ # 'binary-sparc'
+]
mainrepofiles = [
'InRelease',
DIR diff --git a/lib/fs.py b/lib/fs.py
t@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-# copyright (c) 2017 - Ivan J. <parazyd@dyne.org>
-# see LICENSE file for copyright and license details
-
-import config
-
-
-def crawl():
- paths = {}
- for i in range(0, len(config.repos)):
- repo = config.repos[i]["name"]
- basepath = config.repos[i]["dists"]
- sts = []
- for j in config.suites:
- for k in config.suites[j]:
- if config.repos[i]["aliases"] is True:
- if repo in config.aliases:
- try:
- suite = config.aliases[repo][k]
- except:
- if config.repos[i]["skipmissing"] is True:
- continue
- else:
- suite = k
- else:
- suite = k
- skips = ["jessie-security", "ascii-security"] # XXX: HACK:
- if repo == "DEBIAN" and suite in skips:
- continue
- sts.append(suite)
- paths[repo] = sts
- return paths
-
-print(crawl())
DIR diff --git a/lib/log.py b/lib/log.py
t@@ -1,9 +1,10 @@
-#!/usr/bin/env python
-# copyright (c) 2017 - Ivan J. <parazyd@dyne.org>
# see LICENSE file for copyright and license details
-# TODO: Replace with logging
+"""
+Logging functions
+"""
+# TODO: Replace with logging
import sys
DIR diff --git a/lib/net.py b/lib/net.py
t@@ -1,25 +1,30 @@
-#!/usr/bin/env python3
-# copyright (c) 2017 - Ivan J. <parazyd@dyne.org>
# see LICENSE file for copyright and license details
-import requests
+"""
+Network functions/helpers
+"""
+
import os
+import requests
-from .log import die, notice, warn, cleanexit
+from .log import die, warn
def download(url, path):
+ """
+ Downloads a file by providing it an url and a write path
+ """
print("downloading: %s\nto: %s" % (url, path))
r = requests.get(url, stream=True)
if r.status_code == 404:
warn("download of %s failed: not found!" % url)
return
elif r.status_code != 200:
- die("download of %s failed", url)
+ die("download of %s failed" % url)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as f:
- # XXX: should be more on gbit servers
+ # chunk_size {sh,c}ould be more on gbit servers
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
DIR diff --git a/lib/package.py b/lib/package.py
t@@ -1,3 +1,9 @@
+# see LICENSE file for copyright and license details
+
+"""
+Package merging functions and helpers
+"""
+
import os
from gzip import open as gzip_open
DIR diff --git a/lib/parse.py b/lib/parse.py
t@@ -1,22 +1,24 @@
-#!/usr/bin/env python
-# copyright (c) 2017 - Ivan J. <parazyd@dyne.org>
# see LICENSE file for copyright and license details
-import ast
-import gzip
+"""
+Parsing functions/helpers
+"""
+
import re
-#import requests
import time
-from . import config
-from .log import notice
-
def get_time(date):
+ """
+ Gets epoch time
+ """
return time.mktime(time.strptime(date, "%a, %d %b %Y %H:%M:%S %Z"))
def get_date(relfile):
+ """
+ Gets the date from the contents of a Release file
+ """
date = None
contents = relfile.split('\n')
for line in contents:
t@@ -48,6 +50,11 @@ def parse_release(reltext):
def parse_release_re(reltext):
+ """
+ Parses a Release file using regular expressions and returns a dict
+ of the files we keed
+ key = filename, value = sha256 checksum
+ """
_hash = {}
match = re.search('SHA256:+', reltext)
if match:
t@@ -95,9 +102,11 @@ def parse_package_re(entry):
def parse_packages(pkgtext):
- # this parses our package file into a hashmap
- # key: package name, value: entire package paragraph as a hashmap
- map = {}
+ """
+ Parses our package file contents into a hashmap
+ key: package name, value: entire package paragraph as a hashmap
+ """
+ _map = {}
pkgs = pkgtext.split("\n\n")
for pkg in pkgs:
t@@ -105,9 +114,10 @@ def parse_packages(pkgtext):
if m:
line = pkg[m.start():m.end()]
key = line.split(': ')[1]
- map[key] = parse_package(pkg)
+ _map[key] = parse_package(pkg)
+
+ return _map
- return map
def parse_dependencies(dependencies):
"""
t@@ -127,8 +137,8 @@ def parse_dependencies(dependencies):
v = pkg_plus_version.split(' ', 1)
name = v[0]
- # If we get passed an empty string, the name is '', and we just outright
- # stop
+ # If we get passed an empty string, the name is '', and we just
+ # outright stop
if not name:
return {}
t@@ -141,44 +151,12 @@ def parse_dependencies(dependencies):
return r
-def print_package(map, pkgname):
- try:
- pkg = ast.literal_eval(map[pkgname])
- sin = []
- for i in config.pkgfmt:
- if config.pkgfmt[i] in pkg.keys():
- sin.append(config.pkgfmt[i] + pkg[config.pkgfmt[i]])
- return sin
- except:
- log.die("nonexistent package")
-
-
def compare_dict(d1, d2):
+ """
+ Compares two dicts
+ """
d1_keys = set(d1.keys())
d2_keys = set(d2.keys())
intersect_keys = d1_keys.intersection(d2_keys)
modified = {o: (d1[o], d2[o]) for o in intersect_keys if d1[o] != d2[o]}
return modified
-
-
-def compare_release(oldrel, newrel):
- r = requests.get(newrel)
- new = r.text
- with open(oldrel, "rb") as f:
- old = f.read()
-
- oldtime = get_time(get_date(old))
- newtime = get_time(get_date(new))
- if newtime > oldtime:
- notice("Update available")
- newhashes = parse_release(new)
- oldhashes = parse_release(old)
- changes = compare_dict(newhashes, oldhashes)
- # k = pkg name, v = sha256
- return changes
-
-
-# relmap = compare_release("../spool/dists/jessie/updates/Release", "http://security.debian.org/dists/jessie/updates/Release")
-# print relmap
-# for k,v in relmap.iteritems():
-# print(k)