#!/usr/bin/env python3 # -*- encoding: utf-8 -*- import os import json import shutil from .system import subredditTools def readAllFile(s): with open(s) as f: return f.read() wdir = os.path.abspath('.') def main(): idir = os.path.join(wdir, 'i') ddir = os.path.join(wdir, 'd') subreddits = sorted(filter(lambda sr: os.path.isdir(os.path.join(wdir,'r',sr)), os.listdir(os.path.join(wdir,'r')))) subreddits = list(zip( subreddits, map( lambda a: a['links'], map( json.loads, map( readAllFile, map( lambda sr: os.path.join(wdir,'r',sr,'subreddit.json'), subreddits )))), map( lambda sr: os.path.join(wdir,'d',sr), subreddits ) )) linksDown = 0 linksNotDown = 0 linksErr = 0 copyfiles = list() print('\r'+' '*79+'\r'+'Calculating changes...',end='') for subreddit, links, target in subreddits: sdir = os.path.join(ddir,subreddit) for link in links: imgd = os.path.join(idir, link['datakey']) meta = os.path.join(imgd, 'meta.json') if not os.path.exists(meta): linksNotDown+=1 else: files = json.loads(readAllFile(meta)) if len(files)>0: linksDown+=1 for seq, file in enumerate(files): imgfrom = os.path.join(imgd, file['dname']) imgfn = subredditTools.assembleFileName(subreddit, link, seq, file['ext']) imgto = os.path.join(sdir,imgfn) copyfiles.append((imgfrom,imgto)) else: linksErr+=1 del links lcf = len(copyfiles) for (cnt, (src, dst)) in enumerate(copyfiles): if os.path.exists(dst): continue container = os.path.dirname(os.path.abspath(dst)) if not os.path.exists(container): os.makedirs(container) print('\r'+' '*79+'\r'+'%03d%% copied: %05d of %05d'%((((cnt+1)/lcf)*100)//1, cnt+1, lcf),end='') try: shutil.copyfile(src, dst) except KeyboardInterrupt as e: print() print('\r'+' '*79+'\r'+'Deleting interrupted file...',end='') os.remove(dst) print('\r'+' '*79+'\r'+'Aborted safely',end='') print() raise e print('\r'+' '*79+'\r'+'100% copied') print('% 7d files sucessfully downloaded'%lcf) print('% 7d links downloaded'%linksDown) print('% 7d links errored'%linksErr) print('% 7d links ignored'%linksNotDown) print('% 7d links total'%(linksDown+linksNotDown+linksErr)) print() print('Done.') if __name__ == '__main__': main()