Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: getodk/xlsform-online
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v2.12.1
Choose a base ref
...
head repository: getodk/xlsform-online
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
Loading
Showing with 208 additions and 119 deletions.
  1. +7 −0 .gitignore
  2. +21 −0 Dockerfile
  3. +30 −0 README.md
  4. +3 −3 requirements.txt
  5. +1 −1 xlsform_app/templates/upload.html
  6. +4 −4 xlsform_app/urls.py
  7. +129 −83 xlsform_app/views.py
  8. +11 −26 xlsform_prj/settings.py
  9. +2 −2 xlsform_prj/urls.py
7 changes: 7 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/static/
/db.sqlite3
/debug.log
/debug.log.*

__pycache__/
.DS_Store
21 changes: 21 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
FROM python:3.12.5-alpine

RUN apk --update add openjdk8-jre-base git

WORKDIR /usr/src/app

COPY requirements.txt ./

RUN pip install --no-cache-dir -r requirements.txt

COPY . .

EXPOSE 8000

VOLUME ["/tmp_home", "/persistent_home"]

ENV DJANGO_TMP_HOME="/tmp_home"

ENV DJANGO_PERSISTENT_HOME="/persistent_home"

CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--workers", "5", "--timeout", "600", "--max-requests", "25", "--max-requests-jitter", "5", "xlsform_prj.wsgi:application"]
30 changes: 30 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Overview
XLSForm Online is a Django-based web application that uses pyxform to convert a XLSForm to an ODK XForm and shows the preview of the Form in Enketo Express.

# Run locally

## Install requirements
* Python 3
* Java 8

```
pip install --requirement requirements.txt
export DJANGO_SECRET_KEY=<secret value>
export DJANGO_TMP_HOME=<location for temporary Forms>
export DJANGO_PERSISTENT_HOME=<location for permanent Forms>
python3 manage.py runserver
```

# Run in Docker
```
docker build --tag xlsform-online .
docker run --detach --publish 5001:80 xlsform-online
export DJANGO_SECRET_KEY=<secret value>
export DJANGO_TMP_HOME=<location for temporary Forms>
export DJANGO_PERSISTENT_HOME=<location for permanent Forms>
docker run -e DJANGO_SECRET_KEY=$DJANGO_SECRET_KEY -v $DJANGO_TMP_HOME:/tmp_home -v $DJANGO_PERSISTENT_HOME:/persistent_home -p 8000:8000 xlsform-online
```

# CORS

If you want to call `api/xlsform` from another application, please set the `DJANGO_CORS_ALLOWED_ORIGIN` environemnt variable, you can provide comma separated list for allowed origins e.g. `https://staging.enketo.getodk.org, http://localhost:5173`.
6 changes: 3 additions & 3 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
Django==3.2.19
pyxform==1.12.1
gunicorn==20.1.0
Django==4.2.17
pyxform@git+https://github.com/xlsform/pyxform@master
gunicorn==22.0.0
2 changes: 1 addition & 1 deletion xlsform_app/templates/upload.html
Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@
{% if itemsets_url %}
<a href="{{ itemsets_url }}" class="btn btn-success">Download external choices CSV</a>
{% endif %}
<a href="https://enketo.getodk.org/preview?form={{ xml_url | urlencode }}" target="_blank" class="btn btn-success">Preview in browser</a>
<a href="https://staging.enketo.getodk.org/preview?form={{ xml_url | urlencode }}" target="_blank" class="btn btn-success">Preview in browser</a>
</div>
{% endif %}
{% endif %}
8 changes: 4 additions & 4 deletions xlsform_app/urls.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from django.conf.urls import url
from django.urls import re_path
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from . import views

urlpatterns = [
url(r'^json_workbook/', views.json_workbook),
url(r'^$', views.index),
url(r'^downloads/(?P<path>.*)$', views.serve_file),
re_path(r'^$', views.index),
re_path(r'^downloads/(?P<path>.*)$', views.serve_file),
re_path(r'^api/xlsform$', views.api_xlsform),
]

urlpatterns += staticfiles_urlpatterns()
212 changes: 129 additions & 83 deletions xlsform_app/views.py
Original file line number Diff line number Diff line change
@@ -1,124 +1,152 @@
# Create your views here.
from django.http import HttpResponse
from django.http import JsonResponse
from django.http import HttpResponseBadRequest
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.csrf import csrf_exempt
from django.shortcuts import render
from django import forms
from django.conf import settings
from enum import Enum
import shutil

import tempfile
import os
import json
import codecs
import re
import uuid

import pyxform
from pyxform import xls2json
from pyxform.utils import sheet_to_csv, has_external_choices
from pyxform.utils import has_external_choices
from pyxform.xls2json_backends import sheet_to_csv
from pyxform.validators import odk_validate

DJANGO_TMP_HOME = os.environ['DJANGO_TMP_HOME']
DJANGO_PERSISTENT_HOME = os.environ['DJANGO_PERSISTENT_HOME']

class UploadFileForm(forms.Form):
file = forms.FileField()

class PreviewTarget(Enum):
WEB_FORMS = 'web-forms'
ENKETO = 'enketo'

def handle_uploaded_file(f, temp_dir):
def clean_name(name):

# name will be used in a URL and # and , aren't valid characters
return re.sub("#|,","", name)

def append_cors_headers(request, response):
allowed_origin = settings.CORS_ALLOWED_ORIGIN
if(allowed_origin):
origin_list = [item.strip() for item in allowed_origin.split(",")]
request_origin = request.META.get('HTTP_ORIGIN')
if(request_origin in origin_list):
response["Access-Control-Allow-Origin"] = request_origin
response["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"

def handle_uploaded_file(file, directory):

# filename will be used in a URL and # isn't a valid character
filename = f.name.replace("#","")
filename = clean_name(file.name)

xls_path = os.path.join(temp_dir, filename)
destination = open(xls_path, 'wb+')
for chunk in f.chunks():
filepath = os.path.join(directory, filename)
destination = open(filepath, 'wb+')
for chunk in file.chunks():
destination.write(chunk)
destination.close()
return xls_path
return filepath

def copy_file(directory, file_path):
if not (os.access(directory, os.F_OK)):
os.makedirs(directory)
return shutil.copy(file_path, directory)

def json_workbook(request):

def convert_xlsform(file, baseDownloadUrl, preview_target):
error = None
warningsList = []
warnings = None

filename, ext = os.path.splitext(file.name)

filename = clean_name(filename)

if not (os.access(DJANGO_TMP_HOME, os.F_OK)):
os.mkdir(DJANGO_TMP_HOME)

#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(prefix='', dir=DJANGO_TMP_HOME)
form_name = request.POST.get('name', 'form')
output_filename = form_name + '.xml'
out_path = os.path.join(temp_dir, output_filename)
if os.access(out_path, os.F_OK):
os.remove(out_path)
try:
json_survey = xls2json.workbook_to_json(json.loads(request.POST['workbookJson']), form_name=form_name,
warnings=warningsList)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(out_path, warnings=warningsList, pretty_print=False)
except Exception as e:
error = str(e)
return HttpResponse(json.dumps({
'dir': os.path.split(temp_dir)[-1],
'name': output_filename,
'error': error,
'warnings': warningsList,
}, indent=4), mimetype="application/json")

@xframe_options_exempt
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass
dir_uuid = uuid.uuid4().hex
temp_dir = tempfile.mkdtemp(prefix=dir_uuid, dir=DJANGO_TMP_HOME)
relpath_itemsets_csv = None

error = None
warnings = None
# Subdirectory i.e. 'web-form' / 'enketo' is inferred by the source
# api_xlsform is called only by Web Form Preview and the view (upload.html) has only option to view Form in Enketo
permanent_dir = os.path.join(DJANGO_PERSISTENT_HOME, preview_target.value, dir_uuid)
if not (os.access(permanent_dir, os.F_OK)):
os.makedirs(permanent_dir)

filename, ext = os.path.splitext(request.FILES['file'].name)

# filename will be used in a URL and # isn't a valid character
filename = filename.replace("#","")

if not (os.access(DJANGO_TMP_HOME, os.F_OK)):
os.mkdir(DJANGO_TMP_HOME)
try:
if ext == '.xml':
permanent_file_path = handle_uploaded_file(file, permanent_dir)

#Make a randomly generated directory to prevent name collisions
temp_dir = tempfile.mkdtemp(prefix='', dir=DJANGO_TMP_HOME)
# We need to copy the file to temp_dir so that it can be served via `download()`
xml_path = copy_file(temp_dir, permanent_file_path)
relpath = os.path.relpath(xml_path, DJANGO_TMP_HOME)
warnings = odk_validate.check_xform(xml_path)
else:
xml_path = os.path.join(temp_dir, filename + '.xml')
itemsets_url = None

relpath = os.path.relpath(xml_path, DJANGO_TMP_HOME)

#Init the output xml file.
fo = open(xml_path, "wb+")
fo.close()

try:
#TODO: use the file object directly
xls_path = handle_uploaded_file(request.FILES['file'], temp_dir)
warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings, pretty_print=False)

if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(os.path.split(xls_path)[0],
"itemsets.csv")
#TODO: use the file object directly
xls_path = handle_uploaded_file(file, permanent_dir)

warnings = []
json_survey = xls2json.parse_file_to_json(xls_path, warnings=warnings)
survey = pyxform.create_survey_element_from_dict(json_survey)
survey.print_xform_to_file(xml_path, warnings=warnings, pretty_print=False)

if has_external_choices(json_survey):
# Create a csv for the external choices
itemsets_csv = os.path.join(os.path.split(xml_path)[0],
"itemsets.csv")
choices_exported = sheet_to_csv(xls_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings += ["Could not export itemsets.csv, "
"perhaps the external choices sheet is missing."]
else:
relpath_itemsets_csv = os.path.relpath(itemsets_csv, DJANGO_TMP_HOME)
choices_exported = sheet_to_csv(xls_path, itemsets_csv,
"external_choices")
if not choices_exported:
warnings += ["Could not export itemsets.csv, "
"perhaps the external choices sheet is missing."]
else:
itemsets_url = request.build_absolute_uri('./downloads/' + relpath_itemsets_csv)
except Exception as e:
error = 'Error: ' + str(e)
except Exception as e:
error = 'Error: ' + str(e)

return {
'xform_url': None if not relpath else baseDownloadUrl + relpath,
'itemsets_url': None if not relpath_itemsets_csv else baseDownloadUrl + relpath_itemsets_csv,
'error': error,
'warnings': warnings
}

@xframe_options_exempt
def index(request):
if request.method == 'POST': # If the form has been submitted...
form = UploadFileForm(request.POST, request.FILES) # A form bound to the POST data
if form.is_valid(): # All validation rules pass

conversion_result = convert_xlsform(request.FILES['file'], request.build_absolute_uri('./downloads/'), PreviewTarget.ENKETO)

return render(request, 'upload.html', {
'form': UploadFileForm(),
'xml_path': request.build_absolute_uri('./downloads/' + relpath),
'xml_url': request.build_absolute_uri('./downloads/' + relpath),
'itemsets_url': itemsets_url,
'success': not error,
'error': error,
'warnings': warnings,
'xml_path': conversion_result.get('xform_url'),
'xml_url': conversion_result.get('xform_url'),
'itemsets_url': conversion_result.get('itemsets_url'),
'success': not conversion_result.get('error'),
'error': conversion_result.get('error'),
'warnings': conversion_result.get('warnings'),
'result': True,
})
else:
@@ -128,13 +156,31 @@ def index(request):
'form': form,
})


@xframe_options_exempt
def serve_file(request, path):
fo = codecs.open(os.path.join(DJANGO_TMP_HOME, path), mode='r', encoding='utf-8')
data = fo.read()
fo.close()
response = HttpResponse(content_type='application/xml')
response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(os.path.normpath(path))
response.write(data)
return response
path = path.strip("/")
path_segments = path.split("/")
if len(path_segments) == 2 and all(segment not in (".", "..", "") for segment in path_segments):
fo = codecs.open(os.path.join(DJANGO_TMP_HOME, path), mode='r', encoding='utf-8')
data = fo.read()
fo.close()
response = HttpResponse(content_type='application/xml')
response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(os.path.normpath(path))
response.write(data)

append_cors_headers(request, response)

return response

@csrf_exempt
def api_xlsform(request):
if request.method != 'POST':
return HttpResponseBadRequest()

conversion_result = convert_xlsform(request.FILES['file'],request.build_absolute_uri('/downloads/'), PreviewTarget.WEB_FORMS)

response = JsonResponse(conversion_result)

append_cors_headers(request, response)

return response
Loading