Migrate Python 2 to 3 and upgrade Web2py to Version 2.21.1

README.md

@@ -17,13 +17,13 @@ apt-get install python-dev libxml2-dev libxslt1-dev zlib1g-dev
 Install the required packages by running:
 
 ```
-pip install -r requirements.txt
+pip3 install -r requirements.txt
 ```
 
 Also, pip doesn't respect proxy while installing packages from the requirements file. So if you are using a proxy in your terminal you MAY use:
 
 ```
-pip install -r requirements.txt --proxy=<proxy address>
+pip3 install -r requirements.txt --proxy=<proxy address>
 ```
 
 2. To deploy the code, uglify-js and uglifycss needs to be installed
@@ -70,7 +70,7 @@ npm install uglifycss -g
 
    Note: Web2Py does not allow appname to contain hyphens.
 
-4. Install MySQL - [here](http://dev.mysql.com/downloads/)
+4. Install MySQL (v5.6) - [here](http://dev.mysql.com/downloads/)
    Make sure you remember the root password for mysql server.
 
 5. Create a database in MySQL
@@ -119,15 +119,15 @@ npm install uglifycss -g
 
    In case if you want to send emails - Install `postfix` for your respective OS and configure the above smtp server accordingly.
 
-8. Install Redis - [here](https://www.digitalocean.com/community/tutorials/how-to-install-and-secure-redis-on-ubuntu-18-04)
+8. Install and start Redis - [here](https://www.digitalocean.com/community/tutorials/how-to-install-and-secure-redis-on-ubuntu-18-04)
 
 9. Install InfluxDB (1.7) - [here](https://docs.influxdata.com/influxdb/v1.7/introduction/installation/)
 
 10. Navigate back to the web2py folder and start the web2py server.
 
     ```
     $ cd web2py
-    $ python web2py.py -a yourPassword // Choose any password
+    $ python3 web2py.py -a yourPassword // Choose any password
     ```
 
 11. Open the browser and go to the URL -

controllers/appadmin.py

@@ -97,7 +97,7 @@
 
 def get_databases(request):
     dbs = {}
-    for (key, value) in global_env.items():
+    for (key, value) in list(global_env.items()):
         try:
             cond = isinstance(value, GQLDB)
         except:
@@ -321,7 +321,7 @@ def select():
             else:
                 rows = db(query, ignore_common_filters=True).select(
                     *fields, limitby=(start, stop))
-        except Exception, e:
+        except Exception as e:
             import traceback
             tb = traceback.format_exc()
             (rows, nrows) = ([], 0)
@@ -349,7 +349,7 @@ def select():
             import_csv(db[request.vars.table],
                        request.vars.csvfile.file)
             response.flash = T('data uploaded')
-        except Exception, e:
+        except Exception as e:
             response.flash = DIV(T('unable to parse csv file'), PRE(str(e)))
     # end handle upload csv
 
@@ -508,7 +508,7 @@ def GetInHMS(seconds):
         gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
         total.update(gae_stats)
     else:
-        for key, value in cache.ram.storage.iteritems():
+        for key, value in list(cache.ram.storage.items()):
             if isinstance(value, dict):
                 ram['hits'] = value['hit_total'] - value['misses']
                 ram['misses'] = value['misses']
@@ -543,7 +543,7 @@ def GetInHMS(seconds):
                     disk['oldest'] = value[0]
                 disk['keys'].append((key, GetInHMS(time.time() - value[0])))
 
-        ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
+        ram_keys = list(ram.keys()) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
         ram_keys.remove('ratio')
         ram_keys.remove('oldest')
         for key in ram_keys:
@@ -638,14 +638,14 @@ def bg_graph_model():
             meta_graphmodel = dict(group=request.application, color='#ECECEC')
 
         group = meta_graphmodel['group'].replace(' ', '')
-        if not subgraphs.has_key(group):
+        if group not in subgraphs:
             subgraphs[group] = dict(meta=meta_graphmodel, tables=[])
         subgraphs[group]['tables'].append(tablename)
 
         graph.add_node(tablename, name=tablename, shape='plaintext',
                        label=table_template(tablename))
 
-    for n, key in enumerate(subgraphs.iterkeys()):
+    for n, key in enumerate(subgraphs.keys()):
         graph.subgraph(nbunch=subgraphs[key]['tables'],
                     name='cluster%d' % n,
                     style='filled',

controllers/default.py

@@ -236,7 +236,7 @@ def user_editorials():
             # This problem has an official editorial - don't count in leaderboard
             continue
 
-        if user_id_pid_map.has_key((row.user_editorials.user_id, row.user_editorials.problem_id)):
+        if (row.user_editorials.user_id, row.user_editorials.problem_id) in user_id_pid_map:
             # This is to handle multiple accepted editorials of same user on same problem
             continue
 
@@ -256,7 +256,7 @@ def user_editorials():
     for key in user_id_pid_map:
         value = user_id_pid_map[key]
         vote_count = 0 if value.votes.strip() == "" else len(value.votes.split(","))
-        if editorial_count_dict.has_key(value.user_id):
+        if value.user_id in editorial_count_dict:
             update_value = editorial_count_dict[value.user_id]
             update_value["count"] += 1
             update_value["votes"] += vote_count
@@ -838,7 +838,7 @@ def leaderboard():
     cftable = db.custom_friend
 
     global_leaderboard = False
-    if request.vars.has_key("global"):
+    if "global" in request.vars:
         if request.vars["global"] == "True":
             global_leaderboard = True
         else:
@@ -872,14 +872,14 @@ def leaderboard():
     # Do not display blacklisted users in the leaderboard
     aquery &= (atable.blacklisted == False)
 
-    if request.vars.has_key("q") and request.vars["q"]:
-        from urllib import unquote
+    if "q" in request.vars and request.vars["q"]:
+        from urllib.parse import unquote
         institute = unquote(request.vars["q"])
         specific_institute = True
         heading = "StopStalk Leaderboard - " + institute
         aquery &= (atable.institute == institute)
 
-    if request.vars.has_key("country") and request.vars["country"]:
+    if "country" in request.vars and request.vars["country"]:
         country = None
         if request.vars["country"] not in reverse_country_mapping:
             if request.vars["country"] in current.all_countries:
@@ -907,7 +907,7 @@ def leaderboard():
             users = json.loads(user_ratings)
             logged_in_row = None
             if auth.is_logged_in():
-                logged_in_row = filter(lambda x: x[1] == session.handle, users)
+                logged_in_row = [x for x in users if x[1] == session.handle]
                 logged_in_row = None if len(logged_in_row) == 0 else logged_in_row[0]
 
             resp = dict(users=users, logged_in_row=logged_in_row)
@@ -1036,7 +1036,7 @@ def filters():
 
     table = None
     global_submissions = False
-    if get_vars.has_key("global") and get_vars["global"] == "True":
+    if "global" in get_vars and get_vars["global"] == "True":
         global_submissions = True
 
     # If form is not submitted
@@ -1050,7 +1050,7 @@ def filters():
     # these fields should be passed in
     # the URL with empty value
     compulsory_keys = ["pname", "name", "end_date", "start_date"]
-    if set(compulsory_keys).issubset(get_vars.keys()) is False:
+    if set(compulsory_keys).issubset(list(get_vars.keys())) is False:
         session.flash = T("Invalid URL parameters")
         redirect(URL("default", "filters"))
 
@@ -1206,11 +1206,11 @@ def filters():
     def _get_values_list(param_name):
 
         values_list = []
-        if get_vars.has_key(param_name):
+        if param_name in get_vars:
             values_list = get_vars[param_name]
             if isinstance(values_list, str) and values_list != "":
                 values_list = [values_list]
-        elif get_vars.has_key(param_name + "[]"):
+        elif param_name + "[]" in get_vars:
             values_list = get_vars[param_name + "[]"]
             if isinstance(values_list, str):
                 values_list = [values_list]
@@ -1272,7 +1272,7 @@ def mark_friend():
 
     ftable = db.following
     try:
-        friend_id = long(request.args[0])
+        friend_id = int(request.args[0])
     except ValueError:
         raise HTTP(400, "Bad request")
         return T("Invalid user argument!")
@@ -1409,7 +1409,7 @@ def search():
         all_institutes = [x.name.strip("\"") for x in all_institutes]
         all_institutes.append("Other")
 
-        country_name_list = current.all_countries.keys()
+        country_name_list = list(current.all_countries.keys())
         country_name_list.sort()
 
         resp = dict(all_institutes=all_institutes,
@@ -1597,7 +1597,7 @@ def _invalid_url():
         return _invalid_url()
     else:
         try:
-            friend_id = long(request.args[0])
+            friend_id = int(request.args[0])
         except ValueError:
             return _invalid_url()
 
@@ -1700,7 +1700,7 @@ def faq():
             data={"collapsible": "expandable"})
 
     faqs = db(db.faq).select()
-    for i in xrange(len(faqs)):
+    for i in range(len(faqs)):
         li = LI(DIV(B(str(i + 1) + ". " + faqs[i].question),
                     _class="collapsible-header"),
                 DIV(MARKMIN(faqs[i].answer),

controllers/problems.py

@@ -33,7 +33,7 @@ def pie_chart_helper():
     problem_id = int(request.post_vars["pid"])
     submission_type = "friends"
 
-    if request.vars.has_key("submission_type"):
+    if "submission_type" in request.vars:
         if request.vars["submission_type"] == "global":
             submission_type = "global"
         elif request.vars["submission_type"] == "my":
@@ -170,7 +170,7 @@ def add_suggested_tags():
 @auth.requires_login()
 def problem_difficulty():
     if request.env.request_method != "POST" or request.extension != "json":
-        raise(HTTP(405, "Method not allowed"))
+        raise HTTP(405, "Method not allowed")
         return dict()
 
     problem_id = int(request.vars["problem_id"])
@@ -248,7 +248,7 @@ def _represent_tag(tag_id):
 
 # ----------------------------------------------------------------------------
 def get_submissions_list():
-    if request.vars.has_key("problem_id") is False:
+    if ("problem_id" in request.vars) is False:
         # Disables direct entering of a URL
         session.flash = T("Please click on a Problem Link")
         redirect(URL("default", "index"))
@@ -262,7 +262,7 @@ def get_submissions_list():
         return
 
     submission_type = "friends"
-    if request.vars.has_key("submission_type"):
+    if "submission_type" in request.vars:
         if request.vars["submission_type"] == "global":
             submission_type = "global"
         elif request.vars["submission_type"] == "my":
@@ -311,7 +311,7 @@ def index():
     """
     from json import dumps
 
-    if request.vars.has_key("problem_id") is False:
+    if ("problem_id" in request.vars) is False:
         # Disables direct entering of a URL
         session.flash = T("Please click on a Problem Link")
         redirect(URL("default", "index"))
@@ -325,7 +325,7 @@ def index():
         return
 
     submission_type = "friends"
-    if request.vars.has_key("submission_type"):
+    if "submission_type" in request.vars:
         if request.vars["submission_type"] == "global":
             submission_type = "global"
         elif request.vars["submission_type"] == "my":
@@ -459,13 +459,12 @@ def editorials():
     atable = db.auth_user
     query = (uetable.problem_id == record.id)
     user_editorials = db(query).select(orderby=~uetable.added_on)
-    accepted_count = len(filter(lambda x: (x.verification == "accepted" or \
+    accepted_count = len([x for x in user_editorials if (x.verification == "accepted" or \
                                            (auth.is_logged_in() and \
                                             (x.user_id == session.user_id or \
                                              session.user_id in STOPSTALK_ADMIN_USER_IDS)
                                              )
-                                            ),
-                                user_editorials))
+                                            )])
     if accepted_count == 0:
         if auth.is_logged_in():
             table_contents = T("No editorials found! Please contribute to the community by writing an editorial if you've solved the problem.")
@@ -496,12 +495,12 @@ def editorials():
 @auth.requires_login()
 def delete_editorial():
     if len(request.args) < 1:
-        raise(HTTP(400, "Bad request"))
+        raise HTTP(400, "Bad request")
         return
 
     ue_record = db.user_editorials(int(request.args[0]))
     if ue_record is None or session.user_id != ue_record.user_id:
-        raise(HTTP(400, "Bad request"))
+        raise HTTP(400, "Bad request")
         return
 
     if ue_record.verification == "accepted":
@@ -657,7 +656,7 @@ def upload_to_filesystem():
 @auth.requires_login()
 def admin_editorial_approval():
     if session.user_id != 1 or len(request.args) < 2:
-        raise(HTTP(401, "Why are you here ?"))
+        raise HTTP(401, "Why are you here ?")
         return
 
     uetable = db.user_editorials
@@ -993,7 +992,7 @@ def recommendations():
 def update_recommendation_status():
     from recommendations.problems import update_recommendation_status
 
-    pid = long(request.args[0])
+    pid = int(request.args[0])
     update_recommendation_status(session.user_id, pid)
 
 # ==============================================================================

controllers/user.py

@@ -32,7 +32,7 @@ def login_token():
         Only accesible to whitelisted Api Calls
     """
     if not utilities.is_apicall():
-        raise HTTP(400, u'Invalid API params')
+        raise HTTP(400, 'Invalid API params')
     """
         @withparameter email and password return {token : ''} if valid credentials
         @withparameter token returns the new refresh token
@@ -272,7 +272,7 @@ def add_custom_friend():
         redirect(URL("user", "profile", args=original_handle))
 
     # ID of the custom user
-    original_id = long(current_row["id"])
+    original_id = int(current_row["id"])
 
     # Delete this id as this will be incremented
     # automatically on insert
@@ -327,9 +327,9 @@ def get_graph_data():
     graphs = []
     for site in current.SITES:
         lower_site = site.lower()
-        if graph_data.has_key(lower_site + "_data"):
+        if lower_site + "_data" in graph_data:
             graphs.extend(graph_data[lower_site + "_data"])
-    graphs = filter(lambda x: x["data"] != {}, graphs)
+    graphs = [x for x in graphs if x["data"] != {}]
 
     data = dict(graphs=graphs)
     current.REDIS_CLIENT.set(redis_cache_key,
@@ -358,10 +358,12 @@ def update_details():
     stable = db.submission
     record = utilities.get_user_records([session.user_id], "id", "id", True)
 
+    """"
     for field in form_fields:
         if record[field] is None:
             continue
         record[field] = record[field].encode("utf-8")
+    """
 
     # Do not allow to modify stopstalk_handle and email
     atable.stopstalk_handle.writable = False
@@ -461,7 +463,7 @@ def update_friend():
     for field in form_fields:
         if record[field] is None:
             continue
-        record[field] = unicode(record[field], "utf-8").encode("utf-8")
+        record[field] = str(record[field], "utf-8").encode("utf-8")
 
     form = SQLFORM(cftable,
                    record,

deploy_scripts/update_js_timestamp.py

@@ -115,10 +115,10 @@ def convert_to_min(filename):
 
             files = all_js_css_files + all_image_files
 
-        files = map(convert_to_min, files)
+        files = list(map(convert_to_min, files))
 
     if len(files):
         current_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
         for filename in files:
-            print "Updating timestamp for", filename
+            print("Updating timestamp for", filename)
             REDIS_CLIENT.set(filename, current_timestamp)