diff --git a/tools/arm_pack_manager/__init__.py b/tools/arm_pack_manager/__init__.py index b1d1766e8b..4c76af09a8 100644 --- a/tools/arm_pack_manager/__init__.py +++ b/tools/arm_pack_manager/__init__.py @@ -1,17 +1,21 @@ from xdg.BaseDirectory import save_data_path from pycurl import Curl from bs4 import BeautifulSoup -from os.path import join, dirname +from os.path import join, dirname, basename from os import makedirs from errno import EEXIST from threading import Thread from Queue import Queue from re import compile, sub from sys import stderr, stdout +from fuzzywuzzy import process +from itertools import takewhile +import argparse RootPackURL = "http://www.keil.com/pack/index.idx" + protocol_matcher = compile("\w*://") def strip_protocol(url) : return sub(protocol_matcher, "", url) @@ -19,9 +23,9 @@ def strip_protocol(url) : def largest_version(content) : return sorted([t['version'] for t in content.package.releases('release')], reverse=True)[0] -def do_queue(function, interable) : +def do_queue(Class, function, interable) : q = Queue() - threads = [Cacher(q, function) for each in range(20)] + threads = [Class(q, function) for each in range(20)] for each in threads : each.setDaemon(True) each.start() @@ -29,6 +33,16 @@ def do_queue(function, interable) : q.put(thing) q.join() +class Reader (Thread) : + def __init__(self, queue, func) : + Thread.__init__(self) + self.queue = queue + self.func = func + def run(self) : + while True : + url = self.queue.get() + self.func(url) + self.queue.task_done() class Cacher (Thread) : def __init__(self, queue, func) : @@ -73,7 +87,7 @@ class Cache () : curl.setopt(curl.LOW_SPEED_TIME, 2) try : curl.perform() - except Exception as e: + except Exception as e : stderr.write("[ ERROR ] file {} did not download {}\n".format(url, str(e))) self.counter += 1 self.display_counter() @@ -93,12 +107,26 @@ class Cache () : stderr.write("[ ERROR ] {} does not appear to be a conforming .pdsc file\n".format(url)) self.counter += 1 + def get_urls(self): + return [join(pdsc.get('url'), pdsc.get('name')) for pdsc in self.pull_from_cache(RootPackURL).find_all("pdsc")] def cache_everything(self) : - urls = [join(pdsc.get('url'), pdsc.get('name')) for pdsc in self.cache_and_parse(RootPackURL).find_all("pdsc")] - self.total = len(urls) * 2 + self.cache_pack_list(self.get_urls()) + + def cache_descriptors(self) : + self.cache_descriptor_list(self.get_urls()) + + def cache_descriptor_list(self, list) : + self.total = len(list) self.display_counter() - do_queue(self.cache_pdsc_and_pack, urls) + do_queue(Cacher, self.cache_file, list) + stdout.write("\n") + + def cache_pack_list(self, list) : + self.total = len(list) * 2 + self.display_counter() + do_queue(Cacher, self.cache_pdsc_and_pack, list) + stdout.write("\n") def pull_from_cache(self, url) : dest = join(save_data_path('arm-pack-manager'), strip_protocol(url)) @@ -112,6 +140,8 @@ class Cache () : self.cache_file(Curl(), url) return self.pull_from_cache(url) +parser = argparse.ArgumentParser(description='A Handy little utility for keeping your cache of pack files up to date.') +subparsers = parser.add_subparsers(title="Commands") def subcommand(name, *args, **kwargs): def subcommand(command): @@ -132,23 +162,97 @@ def subcommand(name, *args, **kwargs): subparser.add_argument("--no-timeouts", action="store_true", help="Remove all timeouts and try to download unconditionally") def thunk(parsed_args): - cache = Cache(not parsed_args.verbose, parserd_args.no_timeouts) + cache = Cache(not parsed_args.verbose, parsed_args.no_timeouts) argv = [arg['dest'] if 'dest' in arg else arg['name'] for arg in args] argv = [(arg if isinstance(arg, basestring) else arg[-1]).strip('-') for arg in argv] argv = {arg: vars(parsed_args)[arg] for arg in argv if vars(parsed_args)[arg] is not None} - return command(**argv) + return command(cache, **argv) subparser.set_defaults(command=thunk) return command return subcommand +def user_selection (message, options) : + print(message) + for choice, index in zip(options, range(len(options))) : + print("({}) {}".format(index, choice)) + pick = None + while pick is None : + stdout.write("please select an integer from 0 to {} or \"all\"".format(len(options)-1)) + input = raw_input() + try : + if input == "all" : + pick = options + else : + pick = [options[int(input)]] + except ValueError : + print("I did not understand your input") + return pick + +def fuzzy_find(matches, urls) : + choices = {} + for match in matches : + for key, value in process.extract(match, urls, limit=len(urls)) : + choices.setdefault(key, 0) + choices[key] += value + choices = sorted([(v, k) for k, v in choices.iteritems()], reverse=True) + if not choices : return [] + elif len(choices) == 1 : return choices[0][1] + elif choices[0][0] > choices[1][0] : choices = choices[:1] + else : choices = list(takewhile(lambda t: t[0] == choices[0][0], choices)) + return [v for k,v in choices] + +@subcommand('cache', + dict(name='matches', nargs="+", + help="a bunch of things to search for in part names"), + dict(name=["-b","--batch"], action="store_true", + help="don't ask for user input and assume download all")) +def command_cache (cache, matches, batch=False, verbose= False) : + if len(matches) == 1 and matches[0] == "everything" : + cache.cache_everything() + return True + if len(matches) == 1 and matches[0] == "descriptors" : + cache.cache_descriptors() + return True + else : + urls = cache.get_urls() + choices = fuzzy_find(matches, map(basename, urls)) + if not batch and len(choices) > 1 : + choices = user_selection("Please select a file to cache", choices) + to_download = [] + for choice in choices : + for url in urls : + if choice in url : + to_download.append(url) + cache.cache_pack_list(to_download) + return True + +def extract_parts (descriptor) : + devices = descriptor("device") + return dict([(d['dname'], d) for d in devices]) + +@subcommand('find-part', + dict(name='matches', nargs="+", help="words to match to processors"), + dict(name=['-l',"--list_all"], action="store_true", + help="list all cached parts")) +def command_find_part (cache, matches, list_all=False) : + def foo (d) : + try : + parts.update(extract_parts(cache.pull_from_cache(d))) + except AttributeError as e : + print(e) + parts = {} + do_queue(Reader, foo, cache.get_urls()) + choices = fuzzy_find(matches, parts.keys()) + for part in choices : + print part + + if __name__ == "__main__" : - import argparse - parser = argparse.ArgumentParser(description='A Handy little utility for keeping your cache of pack files up to date.') - foo = Cache(True, False) - foo.cache_everything() + args = parser.parse_args() + args.command(args)