|
|
@@ -24,6 +24,7 @@ import OpenSSL.SSL
|
|
|
import subprocess
|
|
|
import requests
|
|
|
import dateutil
|
|
|
+import hashlib
|
|
|
import sqlite3
|
|
|
import math
|
|
|
import time
|
|
|
@@ -440,6 +441,9 @@ def cloud_download(ad_key=None):
|
|
|
url = '{}/{}'.format(cloud_base_url, ad['path'])
|
|
|
response = requests.get(url)
|
|
|
|
|
|
+ if(os.path.isfile(out_file)):
|
|
|
+ return out_file, md5_checksum(out_file)
|
|
|
+
|
|
|
if response.status_code == 200:
|
|
|
hashes = response.headers['x-goog-hash']
|
|
|
hashes = hashes.split(',')
|
|
|
@@ -496,6 +500,13 @@ def send_results(item):
|
|
|
return response
|
|
|
|
|
|
|
|
|
+def md5_checksum(filename):
|
|
|
+ hash_md5 = hashlib.md5()
|
|
|
+ with open(filename, "rb") as f:
|
|
|
+ for chunk in iter(lambda: f.read(4096), b""):
|
|
|
+ hash_md5.update(chunk)
|
|
|
+ return hash_md5.hexdigest()
|
|
|
+
|
|
|
app = setup_endpoint(queue=queue)
|
|
|
loop = IOLoop.current()
|
|
|
loop.add_callback(feed_queue)
|
