docker compose QA, explict requests timeout (#2447)

* Dependency update * Explicit timeout for all requests * Support proxy for all http(s) calls * Optimize jadx download, support system proxy
MobSF · Nov 9, 2024 · 76596e3 · 76596e3
1 parent ee2cb73
commit 76596e3
Show file tree

Hide file tree

Showing 19 changed files with 139 additions and 38 deletions.
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -17,6 +17,7 @@ services:
     restart: always
     ports:
       - "80:4000"
+      - "1337:4001"
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf:ro
     depends_on:
@@ -41,6 +42,8 @@ services:
       - postgres
     networks:
       - mobsf_network
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
 networks:
   mobsf_network:

diff --git a/docker/docker-compose_swarm.yml b/docker/docker-compose_swarm.yml
@@ -19,6 +19,7 @@ services:
     restart: always
     ports:
      - "8000:8000"
+     - "1337:1337"
     volumes:
       - $HOME/MobSF/mobsf_data:/home/mobsf/.MobSF
     environment:
@@ -34,6 +35,8 @@ services:
       - postgres
     networks:
       - mobsf_network
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     secrets:
       - mobsfDB_password
       - mobsf_api_key

diff --git a/docker/nginx.conf b/docker/nginx.conf
@@ -17,4 +17,18 @@ http {
               }
               client_max_body_size 256M;
         }
+        server {
+              listen 4001;
+              location / {
+                proxy_set_header        X-Forwarded-Host    $host;
+                proxy_set_header        X-Forwarded-Port    443;
+                proxy_set_header        X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+                proxy_pass http://mobsf:1337;
+                proxy_redirect         off;
+                proxy_read_timeout     120;
+                proxy_buffering        on;
+              }
+              client_max_body_size 10M;
+        }
 }
diff --git a/mobsf/DynamicAnalyzer/tools/apk_patcher.py b/mobsf/DynamicAnalyzer/tools/apk_patcher.py
@@ -79,7 +79,7 @@ def download_frida_gadget(self, frida_arch, aarch, version):
             return None
         try:
             response = requests.get(f'{settings.FRIDA_SERVER}{version}',
-                                    timeout=3,
+                                    timeout=5,
                                     proxies=proxies,
                                     verify=verify)
             for item in response.json()['assets']:
@@ -90,6 +90,7 @@ def download_frida_gadget(self, frida_arch, aarch, version):
                 return None
             logger.info('Downloading frida-gadget %s', fgadget)
             with requests.get(url,
+                              timeout=5,
                               stream=True,
                               proxies=proxies,
                               verify=verify) as r:

diff --git a/mobsf/DynamicAnalyzer/tools/webproxy.py b/mobsf/DynamicAnalyzer/tools/webproxy.py
@@ -27,8 +27,11 @@ def stop_httptools(url):
         http_proxy = url.replace('https://', 'http://')
         headers = {'httptools': 'kill'}
         url = 'http://127.0.0.1'
-        requests.get(url, headers=headers, proxies={
-                     'http': http_proxy})
+        requests.get(
+            url,
+            timeout=5,
+            headers=headers,
+            proxies={'http': http_proxy})
         logger.info('Killing httptools Proxy')
     except Exception:
         pass

diff --git a/mobsf/DynamicAnalyzer/views/android/frida_server_download.py b/mobsf/DynamicAnalyzer/views/android/frida_server_download.py
@@ -30,13 +30,17 @@ def clean_up_old_binaries(dirc, version):
                 pass
 
 
-def download_frida_server(url, version, fname):
+def download_frida_server(url, version, fname, proxies):
     """Download frida-server-binary."""
     try:
         download_dir = Path(settings.DWD_DIR)
         logger.info('Downloading binary %s', fname)
         dwd_loc = download_dir / fname
-        with requests.get(url, stream=True) as r:
+        with requests.get(
+                url,
+                timeout=5,
+                proxies=proxies,
+                stream=True) as r:
             with LZMAFile(r.raw) as f:
                 with open(dwd_loc, 'wb') as flip:
                     copyfileobj(f, flip)
@@ -62,13 +66,13 @@ def update_frida_server(arch, version):
         logger.exception('[ERROR] Setting upstream proxy')
     try:
         response = requests.get(f'{settings.FRIDA_SERVER}{version}',
-                                timeout=3,
+                                timeout=5,
                                 proxies=proxies,
                                 verify=verify)
         for item in response.json()['assets']:
             if item['name'] == f'{fserver}.xz':
                 url = item['browser_download_url']
-                return download_frida_server(url, version, fserver)
+                return download_frida_server(url, version, fserver, proxies)
         return False
     except Exception:
         logger.exception('[ERROR] Fetching Frida Server Release')

diff --git a/mobsf/DynamicAnalyzer/views/ios/corellium_apis.py b/mobsf/DynamicAnalyzer/views/ios/corellium_apis.py
@@ -54,6 +54,7 @@ def api_ready(self):
         """Check API Availability."""
         try:
             r = requests.get(f'{self.api}/ready',
+                             timeout=5,
                              proxies=self.proxies,
                              verify=self.verify)
             if r.status_code in SUCCESS_RESP:
@@ -73,6 +74,7 @@ def api_auth(self):
             return False
         r = requests.get(
             f'{self.api}/projects',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -89,6 +91,7 @@ def get_projects(self):
             ids = []
             r = requests.get(
                 f'{self.api}/projects?ids_only=true',
+                timeout=5,
                 headers=self.headers,
                 proxies=self.proxies,
                 verify=self.verify)
@@ -104,6 +107,7 @@ def get_authorized_keys(self):
         """Get SSH public keys associated with a project."""
         r = requests.get(
             f'{self.api}/projects/{self.project_id}/keys',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -124,6 +128,7 @@ def add_authorized_key(self, key):
         }
         r = requests.post(
             f'{self.api}/projects/{self.project_id}/keys',
+            timeout=5,
             headers=self.headers,
             json=data,
             proxies=self.proxies,
@@ -149,6 +154,7 @@ def get_instances(self):
         instances = []
         r = requests.get(
             f'{self.api}/instances',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -168,6 +174,7 @@ def create_ios_instance(self, name, flavor, version):
         }
         r = requests.post(
             f'{self.api}/instances',
+            timeout=5,
             headers=self.headers,
             json=data,
             proxies=self.proxies,
@@ -182,6 +189,7 @@ class CorelliumModelsAPI(CorelliumInit):
     def get_models(self):
         r = requests.get(
             f'{self.api}/models',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -202,6 +210,7 @@ def get_supported_os(self, model):
             return False
         r = requests.get(
             f'{self.api}/models/{model}/software',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -223,6 +232,7 @@ def start_instance(self):
         data = {'paused': False}
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/start',
+            timeout=5,
             headers=self.headers,
             json=data,
             proxies=self.proxies,
@@ -238,6 +248,7 @@ def stop_instance(self):
         data = {'soft': True}
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/stop',
+            timeout=5,
             headers=self.headers,
             json=data,
             proxies=self.proxies,
@@ -252,6 +263,7 @@ def unpause_instance(self):
         """Unpause instance."""
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/unpause',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -265,6 +277,7 @@ def reboot_instance(self):
         """Reboot instance."""
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/reboot',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -291,6 +304,7 @@ def poll_instance(self):
         """Check instance status."""
         r = requests.get(
             f'{self.api}/instances/{self.instance_id}',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -306,6 +320,7 @@ def screenshot(self):
             r = requests.get(
                 (f'{self.api}/instances/{self.instance_id}'
                  '/screenshot.png?scale=1'),
+                timeout=5,
                 headers=self.headers,
                 stream=True,
                 proxies=self.proxies,
@@ -322,6 +337,7 @@ def start_network_capture(self):
         """Start network capture."""
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/sslsplit/enable',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -338,6 +354,7 @@ def stop_network_capture(self):
         """Stop network capture."""
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/sslsplit/disable',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -351,6 +368,7 @@ def download_network_capture(self):
         """Download network capture."""
         r = requests.get(
             f'{self.api}/instances/{self.instance_id}/networkMonitor.pcap',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -364,6 +382,7 @@ def console_log(self):
         """Get Console Log."""
         r = requests.get(
             f'{self.api}/instances/{self.instance_id}/consoleLog',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -377,6 +396,7 @@ def get_ssh_connection_string(self):
         """Get SSH connection string."""
         r = requests.get(
             f'{self.api}/instances/{self.instance_id}/quickConnectCommand',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -464,6 +484,7 @@ def device_input(self, event, x, y, max_x, max_y):
                     {'buttons': [], 'wait': 100}]
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/input',
+            timeout=5,
             headers=self.headers,
             json=data,
             proxies=self.proxies,
@@ -485,6 +506,7 @@ def agent_ready(self):
         """Agent ready."""
         r = requests.get(
             f'{self.api}/instances/{self.instance_id}/agent/v1/app/ready',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -500,6 +522,7 @@ def unlock_device(self):
         """Unlock iOS device."""
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/agent/v1/system/unlock',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -533,6 +556,7 @@ def install_ipa(self):
         """Install IPA."""
         r = requests.post(
             f'{self.api}/instances/{self.instance_id}/agent/v1/app/install',
+            timeout=5,
             headers=self.headers,
             json={'path': '/tmp/app.ipa'},
             proxies=self.proxies,
@@ -548,6 +572,7 @@ def run_app(self, bundle_id):
         r = requests.post(
             (f'{self.api}/instances/{self.instance_id}'
              f'/agent/v1/app/apps/{bundle_id}/run'),
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -562,6 +587,7 @@ def stop_app(self, bundle_id):
         r = requests.post(
             (f'{self.api}/instances/{self.instance_id}'
              f'/agent/v1/app/apps/{bundle_id}/kill'),
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -576,6 +602,7 @@ def remove_app(self, bundle_id):
         r = requests.post(
             (f'{self.api}/instances/{self.instance_id}'
              f'/agent/v1/app/apps/{bundle_id}/uninstall'),
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -589,6 +616,7 @@ def list_apps(self):
         """List all apps installed."""
         r = requests.get(
             f'{self.api}/instances/{self.instance_id}/agent/v1/app/apps',
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)
@@ -603,6 +631,7 @@ def get_icons(self, bundleids):
         r = requests.get(
             (f'{self.api}/instances/{self.instance_id}'
              f'/agent/v1/app/icons?{bundleids}'),
+            timeout=5,
             headers=self.headers,
             proxies=self.proxies,
             verify=self.verify)

diff --git a/mobsf/MalwareAnalyzer/views/VirusTotal.py b/mobsf/MalwareAnalyzer/views/VirusTotal.py
@@ -47,6 +47,7 @@ def get_report(self):
             try:
                 response = requests.get(
                     url,
+                    timeout=5,
                     params=params,
                     headers=headers,
                     proxies=proxies,
@@ -100,6 +101,7 @@ def upload_file(self, file_path):
             try:
                 response = requests.post(
                     url,
+                    timeout=5,
                     files=files,
                     data=headers,
                     proxies=proxies,

diff --git a/mobsf/MobSF/init.py b/mobsf/MobSF/init.py
@@ -18,7 +18,7 @@
 
 logger = logging.getLogger(__name__)
 
-VERSION = '4.1.6'
+VERSION = '4.1.7'
 BANNER = r"""
   __  __       _    ____  _____       _  _    _ 
  |  \/  | ___ | |__/ ___||  ___|_   _| || |  / |

diff --git a/mobsf/MobSF/settings.py b/mobsf/MobSF/settings.py
@@ -414,7 +414,6 @@
 
     DOMAIN_MALWARE_SCAN = os.getenv('MOBSF_DOMAIN_MALWARE_SCAN', '1')
     APKID_ENABLED = os.getenv('MOBSF_APKID_ENABLED', '1')
-    QUARK_ENABLED = bool(os.getenv('MOBSF_QUARK_ENABLED', ''))
     # ==================================================
     # ======WINDOWS STATIC ANALYSIS SETTINGS ===========
     # Private key