Fakultas Ilmu Komputer UI

Commit ee85f5d1 authored by Arief Pratama's avatar Arief Pratama
Browse files

[40] Mass upload materi dari Excel

parent a0fa0e08
......@@ -15,7 +15,10 @@
<a class="nav-link" href="{% url 'unggah' %}">
<span>Unggah Materi</span>
</a>
</li>
<a class="nav-link" href="{% url 'unggah_excel' %}">
<span>Unggah Materi (Excel)</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{% url 'dashboard' %}">
......
{% extends 'app/base_dashboard.html' %}
{% load static %}
{% block title %}
<title>Unggah Materi dari Excel | Digipus</title>
{% endblock %}
{% block content %}
<div class="container">
<div class="col-20">
<h1 class="mt-2">Upload Materi dari Excel</h1>
<hr class="mt-0 mb-4">
<form form id="add_form" method="POST" action="" enctype="multipart/form-data">
{% csrf_token %}
<div class="col-md-6">
<div class="fieldWrapper">
<label for="id_excel">File (*.xlsx):</label>
<input type="file"
name="excel"
accept="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
class="form-control"
id="id_excel"
required />
</div>
</div>
{% if messages %}
<div class="col-md-6 mt-2">
{% for message in messages %}
{% if message.level == DEFAULT_MESSAGE_LEVELS.SUCCESS %}
<div class="alert alert-success" role="alert">
{{message}}
</div>
{% endif %}
{% if message.level == DEFAULT_MESSAGE_LEVELS.ERROR %}
<div class="alert alert-danger" role="alert">
{{message}}
</div>
{% endif %}
{% endfor %}
</div>
{% endif %}
<div class="row marl text-center m-3">
<input type="submit" value="Simpan" class="btn btn-success" style="background-color: #615CFD; border-color: #615CFD;" />
</div>
</form>
<div class="col-20">
Template Excel dapat diunduh dari link <a href="{% url 'unggah_excel' %}?template=1">ini</a>.
</div>
</div>
</div>
{% endblock %}
\ No newline at end of file
......@@ -8,6 +8,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile
from django.core.management import call_command
from django.test import Client, TestCase, TransactionTestCase
from django.urls import resolve
from django.contrib import messages as dj_messages
from django.db.utils import IntegrityError
from administration.models import VerificationSetting, VerificationReport
......@@ -18,9 +19,12 @@ from .models import Category, Comment, Materi, Like, Rating, ReqMaterial, Rating
from .views import (DaftarKatalog, DashboardKontributorView, DetailMateri,
ProfilKontributorView, SuksesLoginAdminView,
SuksesLoginKontributorView, SuntingProfilView,
ProfilAdminView, PostsView, SuntingProfilAdminView, RevisiMateriView, ReqMateriView, KatalogPerKontributorView)
ProfilAdminView, PostsView, SuntingProfilAdminView,
RevisiMateriView, ReqMateriView, KatalogPerKontributorView,
UploadMateriExcelView)
from app.forms import SuntingProfilForm
from app.utils.fileManagementUtil import get_random_filename, remove_image_exifdata
import pandas as pd
class DaftarKatalogTest(TestCase):
......@@ -429,6 +433,204 @@ class UploadPageTest(TestCase):
self.assertNotContains(response, "anything")
class UploadExcelPageTest(TestCase):
def setUp(self):
self.client = Client()
self.user = User.objects._create_user(email="kontributor@gov.id",
password="kontributor", is_contributor=True)
def test_upload_excel_page_using_login_func(self):
found = resolve("/unggah_excel/")
self.assertEqual(found.func.__name__,
UploadMateriExcelView.as_view().__name__)
def test_uplaod_excel_page_url_is_exist(self):
# Positive test
self.client.login(email="kontributor@gov.id",
password="kontributor")
response = self.client.get("/unggah_excel/")
self.assertEqual(response.status_code, 200)
# Negative tests
self.client.login(email="kontributor@gov.id",
password="kontributor")
response = Client().get("/fake/")
self.assertEqual(response.status_code, 404)
def test_upload_excel_page_template(self):
url = "/unggah_excel/"
self.client.login(email="kontributor@gov.id",
password="kontributor")
response = self.client.get(url)
expected_template_name = "unggah_excel.html"
self.assertTemplateUsed(response, expected_template_name)
def test_upload_excel_page_title(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
response = self.client.get("/unggah_excel/")
# Positive tests
self.assertContains(response, "Unggah Materi dari Excel")
# Negative tests
self.assertNotContains(response, "Anything")
def test_upload_excel_page_form_field(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
response = self.client.get("/unggah_excel/")
# Positive tests
self.assertContains(response, "File (*.xlsx)")
# Negative tests
self.assertNotContains(response, "anything")
def create_dummy_excel(self, field_lengths={}, categories=[]):
title1 = "Hands-On Machine Learning with Scikit-Learn and TensorFlow: Concepts, Tools, and Techniques to Build Intelligent Systems"
author1 = "Aurelien Geron, Aurelien Geron, Aurelien Geron"
publisher1 = "O'Reilly Media, O'Reilly Media, O'Reilly Media"
categories1 = "Machine Learning,Deep Learning,Computer Science"
description1 = "A series of Deep Learning breakthroughs have boosted the whole field of machine learning over the last decade. Now that machine learning is thriving, even programmers who know close to nothing about this technology can use simple, efficient tools to implement programs capable of learning from data. This practical book shows you how."
if 'title' in field_lengths:
title1 = title1[:field_lengths['title']]
if 'author' in field_lengths:
author1 = author1[:field_lengths['author']]
if 'publisher' in field_lengths:
publisher1 = publisher1[:field_lengths['publisher']]
if len(categories) > 0:
categories1 = ','.join(categories)
data_frame = pd.DataFrame({
'Title': [title1],
'Author': [author1],
'Publisher': [publisher1],
'Categories': [categories1],
'Description': [description1],
})
file_path = os.path.join(settings.MEDIA_ROOT, 'dummy.xlsx')
writer = pd.ExcelWriter(file_path, engine='xlsxwriter') #pylint: disable=abstract-class-instantiated
data_frame.to_excel(writer, index=0)
writer.save()
return file_path, data_frame
def test_upload_excel_upload_file_title_error(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
field_lengths = {
'author':30,
'publisher':30,
}
file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
with open(file_name, 'rb') as fp:
response = self.client.post("/unggah_excel/", {'excel': fp})
messages = list(dj_messages.get_messages(response.wsgi_request))
msg_text = messages[0].message
self.assertIn('Title', msg_text)
def test_upload_excel_upload_file_author_error(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
field_lengths = {
'title':50,
'publisher':30,
}
file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
with open(file_name, 'rb') as fp:
response = self.client.post("/unggah_excel/", {'excel': fp})
messages = list(dj_messages.get_messages(response.wsgi_request))
msg_text = messages[0].message
self.assertIn('Author', msg_text)
def test_upload_excel_upload_file_publisher_error(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
field_lengths = {
'title':50,
'author':30,
}
file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
with open(file_name, 'rb') as fp:
response = self.client.post("/unggah_excel/", {'excel': fp})
messages = list(dj_messages.get_messages(response.wsgi_request))
msg_text = messages[0].message
self.assertIn('Publisher', msg_text)
def test_upload_excel_upload_file_categories_error(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
field_lengths = {
'title':50,
'author':30,
'publisher':30,
}
file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
with open(file_name, 'rb') as fp:
response = self.client.post("/unggah_excel/", {'excel': fp})
messages = list(dj_messages.get_messages(response.wsgi_request))
msg_text = messages[0].message
self.assertIn('Kategori', msg_text)
def test_upload_excel_upload_file_success(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
Category(name='Computer Science').save()
Category(name='Machine Learning').save()
Category(name='Deep Learning').save()
field_lengths = {
'title':50,
'author':30,
'publisher':30,
}
categories = ['Computer Science','Machine Learning','Deep Learning']
file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths, categories=categories)
with open(file_name, 'rb') as fp:
response = self.client.post("/unggah_excel/", {'excel': fp})
title = data_frame['Title'][0]
self.assertTrue(Materi.objects.get(title=title))
def test_upload_excel_download_template(self):
self.client.login(email="kontributor@gov.id",
password="kontributor")
response = self.client.get("/unggah_excel/?template=1")
self.assertEquals(response['Content-Type'],'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
self.assertEquals(response['Content-Disposition'],'attachment; filename=template.xlsx')
class DashboardKontributorViewTest(TestCase):
def setUp(self):
self.client = Client()
......
......@@ -3,7 +3,7 @@ from django.urls import path, re_path
from app import views
from app.views import (DashboardKontributorView, ProfilKontributorView,
SuksesLoginAdminView, SuksesLoginKontributorView,
SuntingProfilView, UploadMateriHTML, UploadMateriView,
SuntingProfilView, UploadMateriHTML, UploadMateriView, UploadMateriExcelView,
ProfilAdminView, PostsView, SuntingProfilAdminView,
ReqMateriView, KatalogPerKontributorView)
......@@ -18,6 +18,7 @@ urlpatterns = [
path("dashboard/", DashboardKontributorView.as_view(), name="dashboard"),
path("revisi/materi/<int:pk>/", views.RevisiMateriView.as_view(), name="revisi"),
path("unggah/", UploadMateriView.as_view(), name="unggah"),
path("unggah_excel/", UploadMateriExcelView.as_view(), name="unggah_excel"),
path("profil/", ProfilKontributorView.as_view(), name="profil"),
path("sunting/", SuntingProfilView.as_view(), name="sunting"),
path("sukses-kontributor/", SuksesLoginKontributorView.as_view(),
......
......@@ -20,6 +20,8 @@ from app.models import Category, Comment, Materi, Like, ViewStatistics, Download
from app.utils.fileManagementUtil import get_random_filename, remove_image_exifdata
from authentication.models import User
import django
import pandas as pd
from io import BytesIO
from django.contrib import messages
......@@ -332,6 +334,105 @@ class UploadMateriHTML(TemplateView):
return template_name
class UploadMateriExcelView(TemplateView):
template_name = "unggah_excel.html"
context = {}
def get_template_names(self):
if self.request.path == "/unggah_excel/":
template_name = "unggah_excel.html"
return template_name
def get(self, request, *args, **kwargs):
if 'template' in self.request.GET:
data_frame = pd.DataFrame({
'Title': [],
'Author': [],
'Publisher': [],
'Categories': [],
'Description': [],
})
with BytesIO() as b:
writer = pd.ExcelWriter(b, engine='xlsxwriter') #pylint: disable=abstract-class-instantiated
data_frame.to_excel(writer, index=0)
writer.save()
response = HttpResponse(
b.getvalue(),
content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
response["Content-Disposition"] = "attachment; filename=template.xlsx"
return response
else:
context = self.get_context_data(**kwargs)
return self.render_to_response(context)
def post(self, request, *args, **kwargs):
excel_file = request.FILES['excel']
excel = pd.read_excel(excel_file)
row,lines = excel.shape
categories = Category.objects.all()
field_length = {
'title' : 50,
'author' : 30,
'publisher' : 30,
}
message = None
# First pass, validate input
for i in range(row):
# Validate Categories
for c in excel['Categories'][i].split(","):
sel_cat = categories.filter(name=c)
if sel_cat.count() == 0:
message = f"Kategori %s tidak ditemukan" % c
break
if len(excel['Title'][i]) > field_length['title']:
message = f"Title maksimal %d karakter" % field_length['title']
if len(excel['Author'][i]) > field_length['author']:
message = f"Author maksimal %d karakter" % field_length['author']
if len(excel['Publisher'][i]) > field_length['publisher']:
message = f"Publisher maksimal %d karakter" % field_length['publisher']
if message != None:
break
if message != None:
messages.error(request, message)
return HttpResponseRedirect('/unggah_excel/')
# Second pass, save data
with django.db.transaction.atomic():
for i in range(row):
materi = Materi(
title=excel['Title'][i],
author=excel['Author'][i],
publisher=excel['Publisher'][i],
descriptions=excel['Description'][i],
uploader=request.user
)
materi.save()
for c in excel['Categories'][i].split(","):
materi.categories.add(categories.get(name=c))
messages.success(request, 'Materi berhasil diunggah')
return HttpResponseRedirect('/unggah_excel/')
class DashboardKontributorView(TemplateView):
template_name = "dashboard.html"
......
......@@ -44,7 +44,9 @@ lazy-object-proxy==1.4.3
mccabe==0.6.1
more-itertools==8.2.0
nodeenv==1.3.5
numpy==1.19.2
packaging==20.3
pandas==1.1.3
pathspec==0.8.0
Pillow==7.1.1
pluggy==0.13.1
......@@ -82,4 +84,6 @@ virtualenv==20.0.18
wcwidth==0.1.9
whitenoise==5.0.1
wrapt==1.11.2
xlrd==1.2.0
XlsxWriter==1.3.6
zipp==3.1.0
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment