diff --git a/tools/arm_pack_manager/__init__.py b/tools/arm_pack_manager/__init__.py index e3a0b4eee5..779cf84536 100644 --- a/tools/arm_pack_manager/__init__.py +++ b/tools/arm_pack_manager/__init__.py @@ -6,55 +6,88 @@ from os import makedirs from errno import EEXIST from threading import Thread from Queue import Queue +from re import compile, sub RootPackURL = "http://www.keil.com/pack/index.idx" class Cacher (Thread) : - def __init__(self, queue, silent) : + def __init__(self, queue, silent, func) : Thread.__init__(self) self.queue = queue self.curl = Curl() self.curl.setopt(self.curl.FOLLOWLOCATION, True) self.silent = silent + self.func = func def run(self) : while True : url = self.queue.get() - cache_file(self.curl, url, self.silent) + self.func(self.curl, url, self.silent) self.queue.task_done() +protocol_matcher = compile("\w*://") +def strip_protocol(url) : + return sub(protocol_matcher, "", url) + def cache_file (curl, url, silent=False) : if not silent : print("Caching {}...".format(url)) - dest = join(save_data_path('arm-pack-manager'), url) + dest = join(save_data_path('arm-pack-manager'), strip_protocol(url)) try : makedirs(dirname(dest)) except OSError as exc : if exc.errno == EEXIST : pass else : raise with open(dest, "wb+") as fd : - curl.setopt(curl.URL, RootPackURL) + curl.setopt(curl.URL, url) curl.setopt(curl.FOLLOWLOCATION, True) curl.setopt(curl.WRITEDATA, fd) - curl.perform() + curl.setopt(curl.TIMEOUT, 5) + try : + curl.perform() + except Exception: + pass -def cache_everything(silent=False) : - url_queue = Queue() - urls = [pdsc.get('url') + pdsc.get('name') for pdsc in cache_and_parse(RootPackURL,silent).find_all("pdsc")] - threads = [Cacher(url_queue, silent) for each in range(10)] +def largest_version(content) : + return sorted([t['version'] for t in content.package.releases('release')], reverse=True)[0] + +def cache_pdsc_and_pack (curl, url, silent=False) : + content = cache_and_parse(url) + try : + new_url = content.package.url.get_text() + if not new_url.endswith("/") : + new_url = new_url + "/" + cache_file(curl, + new_url + + content.package.vendor.get_text() + "." + + content.package.find('name').get_text() + "." + + largest_version(content) + ".pack", + silent) + except AttributeError : + print("[WARNING] {} does not appear to be a conforming .pdsc file".format(url)) + +def do_queue(function, interable, silent=False) : + q = Queue() + threads = [Cacher(q, silent, function) for each in range(10)] for each in threads : each.setDaemon(True) each.start() - for url in urls: - url_queue.put(url) - url_queue.join() + for thing in interable : + q.put(thing) + q.join() +def cache_everything(silent=False) : + urls = [join(pdsc.get('url'), pdsc.get('name')) for pdsc in cache_and_parse(RootPackURL,silent).find_all("pdsc")] + do_queue(cache_pdsc_and_pack, urls, silent) def pull_from_cache(url) : - dest = join(save_data_path('arm-pack-manager'), url) + dest = join(save_data_path('arm-pack-manager'), strip_protocol(url)) with open(dest, "r") as fd : return BeautifulSoup(fd, "html.parser") +def gen_dict_from_cache() : + pdsc_files = pull_from_cache(RootPackUrl) + def cache_and_parse(url, silent=False) : cache_file(Curl(), url, silent) return pull_from_cache(url)