Skip to content

Commit 4ff6847

Browse files
author
marcobizzarr1
committed
patched bulk download API; refactoring
1 parent aad980c commit 4ff6847

File tree

6 files changed

+127
-53
lines changed

6 files changed

+127
-53
lines changed

‎README.md‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,15 @@ md5_list = [
5656
]
5757

5858
result = sbx.bulk_download_request(md5_list=md5_list, api_key="my-api-key")
59-
print result
6059
if result.status == SUCCESS:
61-
print sbx.bulk_download_retrieve(id_request=result.msg['id_request'], api_key="my-api-key", path="output\\directory\\")
60+
print result
61+
while True:
62+
result2 = sbx.bulk_download_retrieve(id_request=result.msg['id_request'], api_key="my-api-key", path="output\\directory\\")
63+
if result2.status != PROCESSING:
64+
print result2
65+
break
66+
67+
time.sleep(1)
6268
```
6369

6470
To retrieve scan result of a specific MD5
@@ -174,29 +180,41 @@ print result
174180

175181
# More advanced usage examples
176182

177-
Find all domains registered in the last 7 days, print out the malware tags related to them and
183+
Find all domains registered in the last 3 days, print out the malware tags related to them and
178184
list all MD5 samples connecting to them. Then for each one of the samples retrieve the matched
179185
behavioral rules
180186

181187
```python
182188
from deepviz import intel, sandbox
183189
API_KEY = "0000000000000000000000000000000000000000000000000000000000000000"
184190
ThreatIntel = intel.Intel()
185-
ThreatSbx = sandbox.Sandbox()
186-
result_domains = ThreatIntel.domain_info(api_key=API_KEY, time_delta="7d")
187-
domains = result_domains.msg
188-
for domain in domains.keys():
189-
result_list_samples = ThreatIntel.advanced_search(api_key=API_KEY, domain=[domain], classification="M")
190-
if isinstance(result_list_samples.msg, list):
191-
if len(domains[domain]['tag']):
192-
print "DOMAIN: %s ==> %s samples [TAG: %s]" % (domain, len(result_list_samples.msg), ", ".join((tag['key'] for tag in domains[domain]['tag'])))
191+
ThreatSbx = Sandbox()
192+
result_domains = ThreatIntel.domain_info(api_key=API_KEY, time_delta="3d")
193+
if result_domains.status == SUCCESS:
194+
domains = result_domains.msg
195+
for domain in domains.keys():
196+
result_list_samples = ThreatIntel.advanced_search(api_key=API_KEY, domain=[domain], classification="M")
197+
if result_list_samples.status == SUCCESS:
198+
if isinstance(result_list_samples.msg, list):
199+
if len(domains[domain]['tag']):
200+
print "DOMAIN: %s ==> %s samples [TAG: %s]" % (domain, len(result_list_samples.msg), ", ".join((tag for tag in domains[domain]['tag'])))
201+
else:
202+
print "DOMAIN: %s ==> %s samples" % (domain, len(result_list_samples.msg))
203+
204+
for sample in result_list_samples.msg:
205+
result_report = ThreatSbx.sample_report(md5=sample, api_key=API_KEY, filters=["rules"])
206+
if result_report.status == SUCCESS:
207+
print "%s => [%s]" % (sample, ", ".join([rule for rule in result_report.msg['rules']]))
208+
else:
209+
print result_report
210+
break
211+
else:
212+
print "DOMAIN: %s ==> No samples found" % domain
193213
else:
194-
print "DOMAIN: %s ==> %s samples" % (domain, len(result_list_samples.msg))
195-
for sample in result_list_samples.msg:
196-
result_report = ThreatSbx.sample_report(md5=sample, api_key=API_KEY, filters=["rules"])
197-
print "%s => [%s]" % (sample, ", ".join((rule for rule in result_report.msg['rules'])))
198-
else:
199-
print "DOMAIN: %s ==> No samples found" % domain
214+
print result_list_samples
215+
break
216+
else:
217+
print result_domains
200218
```
201219
result:
202220

‎deepviz/intel.py‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,14 @@ def ip_info(self, api_key=None, ip=None, time_delta=None, history=False):
5959
except Exception as e:
6060
return Result(status=NETWORK_ERROR, msg="Error while connecting to Deepviz: %s" % e)
6161

62-
data = json.loads(r.content)
62+
try:
63+
data = json.loads(r.content)
64+
except Exception as e:
65+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
6366

6467
if r.status_code == 200:
6568
return Result(status=SUCCESS, msg=data['data'])
6669
else:
67-
data = json.loads(r.content)
6870
if r.status_code >= 500:
6971
return Result(status=SERVER_ERROR, msg="{status_code} - Error while connecting to Deepviz: {errmsg}".format(status_code=r.status_code, errmsg=data['errmsg']))
7072
else:
@@ -137,7 +139,10 @@ def domain_info(self, api_key=None, domain=None, time_delta=None, history=False,
137139
msg = "Error while connecting to Deepviz: %s" % e
138140
return Result(status=NETWORK_ERROR, msg=msg)
139141

140-
data = json.loads(r.content)
142+
try:
143+
data = json.loads(r.content)
144+
except Exception as e:
145+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
141146

142147
if r.status_code == 200:
143148
return Result(status=SUCCESS, msg=data['data'])
@@ -177,7 +182,10 @@ def search(self, api_key=None, search_string=None, start_offset=None, elements=N
177182
except Exception as e:
178183
return Result(status=NETWORK_ERROR, msg="Error while connecting to Deepviz: %s" % e)
179184

180-
data = json.loads(r.content)
185+
try:
186+
data = json.loads(r.content)
187+
except Exception as e:
188+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
181189

182190
if r.status_code == 200:
183191
return Result(status=SUCCESS, msg=data['data'])
@@ -224,7 +232,10 @@ def advanced_search(self, api_key=None, sim_hash=None, created_files=None, imp_h
224232
except Exception as e:
225233
return Result(status=NETWORK_ERROR, msg="Error while connecting to Deepviz: %s" % e)
226234

227-
data = json.loads(r.content)
235+
try:
236+
data = json.loads(r.content)
237+
except Exception as e:
238+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
228239

229240
if r.status_code == 200:
230241
msg = data['data']

‎deepviz/result.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
CLIENT_ERROR = "DEEPVIZ_STATUS_CLIENT_ERROR" # Http 4xx
55
NETWORK_ERROR = "DEEPVIZ_STATUS_NETWORK_ERROR" # Cannot contact Deepviz
66
INTERNAL_ERROR = "DEEPVIZ_STATUS_INTERNAL_ERROR"
7+
PROCESSING = "DEEPVIZ_STATUS_PROCESSING" # Result is not ready yet
78

89

910
class Result:

‎deepviz/sandbox.py‎

Lines changed: 40 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,15 @@ def upload_sample(self, path=None, api_key=None):
5555
msg = "Error while connecting to Deepviz: %s" % e
5656
return Result(status=NETWORK_ERROR, msg=msg)
5757

58-
if r.status_code == 200:
58+
try:
5959
data = json.loads(r.content)
60+
except Exception as e:
61+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
62+
63+
if r.status_code == 200:
6064
msg = data['data']
6165
return Result(status=SUCCESS, msg=msg)
6266
else:
63-
data = json.loads(r.content)
6467
if r.status_code >= 500:
6568
return Result(status=SERVER_ERROR, msg="{status_code} - Error while connecting to Deepviz: {errmsg}".format(status_code=r.status_code, errmsg=data['errmsg']))
6669
else:
@@ -119,10 +122,10 @@ def download_sample(self, md5=None, path=None, api_key=None):
119122
return Result(status=INTERNAL_ERROR, msg=msg)
120123

121124
body = json.dumps(
122-
{
123-
"api_key": api_key,
124-
"md5": md5
125-
})
125+
{
126+
"api_key": api_key,
127+
"md5": md5
128+
})
126129
try:
127130
r = requests.post(URL_DOWNLOAD_SAMPLE, data=body)
128131
except Exception as e:
@@ -133,7 +136,11 @@ def download_sample(self, md5=None, path=None, api_key=None):
133136
_file.close()
134137
return Result(status=SUCCESS, msg="Sample downloaded to '%s'" % finalpath)
135138
else:
136-
data = json.loads(r.content)
139+
try:
140+
data = json.loads(r.content)
141+
except Exception as e:
142+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
143+
137144
if r.status_code >= 500:
138145
return Result(status=SERVER_ERROR, msg="{status_code} - Error while connecting to Deepviz: {errmsg}".format(status_code=r.status_code, errmsg=data['errmsg']))
139146
else:
@@ -159,7 +166,10 @@ def sample_result(self, md5=None, api_key=None):
159166
except Exception as e:
160167
return Result(status=NETWORK_ERROR, msg="Error while connecting to Deepviz: %s" % e)
161168

162-
data = json.loads(r.content)
169+
try:
170+
data = json.loads(r.content)
171+
except Exception as e:
172+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
163173

164174
if r.status_code == 200:
165175
return Result(status=SUCCESS, msg=data['data'])
@@ -198,7 +208,10 @@ def sample_report(self, md5=None, api_key=None, filters=None):
198208
except Exception as e:
199209
return Result(status=NETWORK_ERROR, msg="Error while connecting to Deepviz: %s" % e)
200210

201-
data = json.loads(r.content)
211+
try:
212+
data = json.loads(r.content)
213+
except Exception as e:
214+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
202215

203216
if r.status_code == 200:
204217
return Result(status=SUCCESS, msg=data['data'])
@@ -227,7 +240,10 @@ def bulk_download_request(self, md5_list=None, api_key=None):
227240
msg = "Error while connecting to Deepviz. [%s]" % e
228241
return Result(status=NETWORK_ERROR, msg=msg)
229242

230-
data = json.loads(r.content)
243+
try:
244+
data = json.loads(r.content)
245+
except Exception as e:
246+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
231247

232248
if r.status_code == 200:
233249
return Result(status=SUCCESS, msg=data['data'])
@@ -261,21 +277,30 @@ def bulk_download_retrieve(self, id_request=None, path=None, api_key=None):
261277
return Result(status=INTERNAL_ERROR, msg="Cannot create file '%s'" % finalpath)
262278

263279
body = json.dumps(
264-
{
265-
"api_key": api_key,
266-
"id_request": str(id_request)
267-
})
280+
{
281+
"api_key": api_key,
282+
"id_request": str(id_request)
283+
})
268284
try:
269285
r = requests.post(URL_DOWNLOAD_BULK, data=body)
270286
except Exception as e:
287+
_file.close()
271288
return Result(status=NETWORK_ERROR, msg="Error while connecting to Deepviz: %s" % e)
272289

273290
if r.status_code == 200:
274291
_file.write(r.content)
275292
_file.close()
276293
return Result(status=SUCCESS, msg="File downloaded to '%s'" % finalpath)
294+
elif r.status_code == 428:
295+
_file.close()
296+
return Result(status=PROCESSING, msg="{status_code} - Your request is being processed. Please try again in a few minutes".format(status_code=r.status_code))
277297
else:
278-
data = json.loads(r.content)
298+
_file.close()
299+
try:
300+
data = json.loads(r.content)
301+
except Exception as e:
302+
return Result(status=INTERNAL_ERROR, msg="Error loading Deepviz response: %s" % e)
303+
279304
if r.status_code >= 500:
280305
return Result(status=SERVER_ERROR, msg="{status_code} - Error while connecting to Deepviz: {errmsg}".format(status_code=r.status_code, errmsg=data['errmsg']))
281306
else:

‎examples/sandbox_test.py‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,16 @@
5050
]
5151

5252
result = sbx.bulk_download_request(md5_list=md5_list, api_key=API_KEY)
53-
print result
5453
if result.status == SUCCESS:
55-
print sbx.bulk_download_retrieve(id_request=result.msg['id_request'], api_key=API_KEY, path=".")
54+
print result
55+
while True:
56+
result2 = sbx.bulk_download_retrieve(id_request=result.msg['id_request'], api_key=API_KEY, path=".")
57+
if result2.status != PROCESSING:
58+
print result2
59+
break
60+
61+
time.sleep(1)
62+
5663

5764
########################################################################################################################
5865

@@ -90,17 +97,29 @@
9097
# behavioral rules
9198

9299
ThreatSbx = Sandbox()
93-
result_domains = ThreatIntel.domain_info(api_key=API_KEY, time_delta="7d")
94-
domains = result_domains.msg
95-
for domain in domains.keys():
96-
result_list_samples = ThreatIntel.advanced_search(api_key=API_KEY, domain=[domain], classification="M")
97-
if isinstance(result_list_samples.msg, list):
98-
if len(domains[domain]['tag']):
99-
print "DOMAIN: %s ==> %s samples [TAG: %s]" % (domain, len(result_list_samples.msg), ", ".join((tag for tag in domains[domain]['tag'])))
100+
result_domains = ThreatIntel.domain_info(api_key=API_KEY, time_delta="3d")
101+
if result_domains.status == SUCCESS:
102+
domains = result_domains.msg
103+
for domain in domains.keys():
104+
result_list_samples = ThreatIntel.advanced_search(api_key=API_KEY, domain=[domain], classification="M")
105+
if result_list_samples.status == SUCCESS:
106+
if isinstance(result_list_samples.msg, list):
107+
if len(domains[domain]['tag']):
108+
print "DOMAIN: %s ==> %s samples [TAG: %s]" % (domain, len(result_list_samples.msg), ", ".join((tag for tag in domains[domain]['tag'])))
109+
else:
110+
print "DOMAIN: %s ==> %s samples" % (domain, len(result_list_samples.msg))
111+
112+
for sample in result_list_samples.msg:
113+
result_report = ThreatSbx.sample_report(md5=sample, api_key=API_KEY, filters=["rules"])
114+
if result_report.status == SUCCESS:
115+
print "%s => [%s]" % (sample, ", ".join([rule for rule in result_report.msg['rules']]))
116+
else:
117+
print result_report
118+
break
119+
else:
120+
print "DOMAIN: %s ==> No samples found" % domain
100121
else:
101-
print "DOMAIN: %s ==> %s samples" % (domain, len(result_list_samples.msg))
102-
for sample in result_list_samples.msg:
103-
result_report = ThreatSbx.sample_report(md5=sample, api_key=API_KEY, filters=["rules"])
104-
print "%s => [%s]" % (sample, ", ".join((rule for rule in result_report.msg['rules'])))
105-
else:
106-
print "DOMAIN: %s ==> No samples found" % domain
122+
print result_list_samples
123+
break
124+
else:
125+
print result_domains

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
setuptools.setup(
44
name='python-deepviz',
5-
version='1.1.1',
5+
version='1.1.2',
66
author='Saferbytes',
77
author_email='info@saferbytes.it',
88
url="https://github.com/saferbytes/python-deepviz",

0 commit comments

Comments
 (0)