[40] Mass upload materi dari Excel

app/templates/app/includes/sidebar.html

@@ -15,7 +15,10 @@
         <a class="nav-link" href="{% url 'unggah' %}">
           <span>Unggah Materi</span>
         </a>
-    </li>
+        <a class="nav-link" href="{% url 'unggah_excel' %}">
+            <span>Unggah Materi (Excel)</span>
+          </a>
+      </li>
 
     <li class="nav-item">
         <a class="nav-link" href="{% url 'dashboard' %}">

app/templates/unggah_excel.html

+{% extends 'app/base_dashboard.html' %}
+{% load static %}
+
+{% block title %}
+<title>Unggah Materi dari Excel | Digipus</title>
+{% endblock %}
+
+{% block content %}
+<div class="container">
+    <div class="col-20">
+        <h1 class="mt-2">Upload Materi dari Excel</h1>
+        <hr class="mt-0 mb-4">
+
+        <form form id="add_form" method="POST" action="" enctype="multipart/form-data">
+            {% csrf_token %}
+
+            <div class="col-md-6">
+                <div class="fieldWrapper">
+                    <label for="id_excel">File (*.xlsx):</label> 
+                    <input type="file" 
+                        name="excel" 
+                        accept="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" 
+                        class="form-control" 
+                        id="id_excel" 
+                        required />
+                </div>
+            </div>
+            
+            {% if messages %}
+            <div class="col-md-6  mt-2">
+                {% for message in messages %}
+                    {% if message.level == DEFAULT_MESSAGE_LEVELS.SUCCESS %}
+                    <div class="alert alert-success" role="alert">
+                    {{message}}
+                    </div>
+                    {% endif %}
+                    {% if message.level == DEFAULT_MESSAGE_LEVELS.ERROR %}
+                    <div class="alert alert-danger" role="alert">
+                    {{message}}
+                    </div>
+                    {% endif %}
+                {% endfor %}
+            </div>
+            {% endif %}
+
+            <div class="row marl text-center m-3">
+                <input type="submit" value="Simpan" class="btn btn-success" style="background-color: #615CFD; border-color: #615CFD;" />
+            </div>
+        </form>
+
+        <div class="col-20">
+            Template Excel dapat diunduh dari link <a href="{% url 'unggah_excel' %}?template=1">ini</a>.
+        </div>
+
+    </div>
+</div>
+{% endblock %}
\ No newline at end of file

app/tests.py

@@ -8,6 +8,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.management import call_command
 from django.test import Client, TestCase, TransactionTestCase
 from django.urls import resolve
+from django.contrib import messages as dj_messages
 from django.db.utils import IntegrityError
 
 from administration.models import VerificationSetting, VerificationReport
@@ -18,9 +19,12 @@ from .models import Category, Comment, Materi, Like, Rating, ReqMaterial, Rating
 from .views import (DaftarKatalog, DashboardKontributorView, DetailMateri,
                     ProfilKontributorView, SuksesLoginAdminView,
                     SuksesLoginKontributorView, SuntingProfilView,
-                    ProfilAdminView, PostsView, SuntingProfilAdminView, RevisiMateriView, ReqMateriView, KatalogPerKontributorView)
+                    ProfilAdminView, PostsView, SuntingProfilAdminView, 
+                    RevisiMateriView, ReqMateriView, KatalogPerKontributorView,
+                    UploadMateriExcelView)
 from app.forms import SuntingProfilForm
 from app.utils.fileManagementUtil import get_random_filename, remove_image_exifdata
+import pandas as pd
 
 
 class DaftarKatalogTest(TestCase):
@@ -429,6 +433,204 @@ class UploadPageTest(TestCase):
         self.assertNotContains(response, "anything")
 
 
+class UploadExcelPageTest(TestCase):
+    def setUp(self):
+        self.client = Client()
+        self.user = User.objects._create_user(email="kontributor@gov.id",
+                                              password="kontributor", is_contributor=True)
+
+    def test_upload_excel_page_using_login_func(self):
+        found = resolve("/unggah_excel/")
+        self.assertEqual(found.func.__name__,
+                         UploadMateriExcelView.as_view().__name__)
+
+    def test_uplaod_excel_page_url_is_exist(self):
+        # Positive test
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+        response = self.client.get("/unggah_excel/")
+        self.assertEqual(response.status_code, 200)
+
+        # Negative tests
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+        response = Client().get("/fake/")
+        self.assertEqual(response.status_code, 404)
+
+    def test_upload_excel_page_template(self):
+        url = "/unggah_excel/"
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+        response = self.client.get(url)
+        expected_template_name = "unggah_excel.html"
+        self.assertTemplateUsed(response, expected_template_name)
+
+    def test_upload_excel_page_title(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+        response = self.client.get("/unggah_excel/")
+
+        # Positive tests
+        self.assertContains(response, "Unggah Materi dari Excel")
+
+        # Negative tests
+        self.assertNotContains(response, "Anything")
+
+    def test_upload_excel_page_form_field(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+        response = self.client.get("/unggah_excel/")
+
+        # Positive tests
+        self.assertContains(response, "File (*.xlsx)")
+
+        # Negative tests
+        self.assertNotContains(response, "anything")
+
+    def create_dummy_excel(self, field_lengths={}, categories=[]):
+        title1 = "Hands-On Machine Learning with Scikit-Learn and TensorFlow: Concepts, Tools, and Techniques to Build Intelligent Systems"
+        author1 = "Aurelien Geron, Aurelien Geron, Aurelien Geron"
+        publisher1 = "O'Reilly Media, O'Reilly Media, O'Reilly Media"
+        categories1 = "Machine Learning,Deep Learning,Computer Science"
+        description1 = "A series of Deep Learning breakthroughs have boosted the whole field of machine learning over the last decade. Now that machine learning is thriving, even programmers who know close to nothing about this technology can use simple, efficient tools to implement programs capable of learning from data. This practical book shows you how."
+
+        if 'title' in field_lengths:
+            title1 = title1[:field_lengths['title']]
+
+        if 'author' in field_lengths:
+            author1 = author1[:field_lengths['author']]
+
+        if 'publisher' in field_lengths:
+            publisher1 = publisher1[:field_lengths['publisher']]
+
+        if len(categories) > 0:
+            categories1 = ','.join(categories)
+
+        data_frame = pd.DataFrame({
+            'Title': [title1],
+            'Author': [author1],
+            'Publisher': [publisher1],
+            'Categories': [categories1],
+            'Description': [description1],
+        })
+
+        file_path = os.path.join(settings.MEDIA_ROOT, 'dummy.xlsx')
+
+        writer = pd.ExcelWriter(file_path, engine='xlsxwriter') #pylint: disable=abstract-class-instantiated
+        data_frame.to_excel(writer, index=0)
+        writer.save()
+
+        return file_path, data_frame
+
+    def test_upload_excel_upload_file_title_error(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+
+        field_lengths = {
+            'author':30,
+            'publisher':30,
+        }
+        file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
+
+        with open(file_name, 'rb') as fp:
+            response = self.client.post("/unggah_excel/", {'excel': fp})
+        
+        messages = list(dj_messages.get_messages(response.wsgi_request))
+        msg_text = messages[0].message
+
+        self.assertIn('Title', msg_text)
+
+    def test_upload_excel_upload_file_author_error(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+
+        field_lengths = {
+            'title':50,
+            'publisher':30,
+        }
+        file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
+
+        with open(file_name, 'rb') as fp:
+            response = self.client.post("/unggah_excel/", {'excel': fp})
+        
+        messages = list(dj_messages.get_messages(response.wsgi_request))
+        msg_text = messages[0].message
+
+        self.assertIn('Author', msg_text)
+
+
+    def test_upload_excel_upload_file_publisher_error(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+
+        field_lengths = {
+            'title':50,
+            'author':30,
+        }
+        file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
+
+        with open(file_name, 'rb') as fp:
+            response = self.client.post("/unggah_excel/", {'excel': fp})
+        
+        messages = list(dj_messages.get_messages(response.wsgi_request))
+        msg_text = messages[0].message
+
+        self.assertIn('Publisher', msg_text)
+
+    def test_upload_excel_upload_file_categories_error(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+
+        field_lengths = {
+            'title':50,
+            'author':30,
+            'publisher':30,
+        }
+        file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths)
+
+        with open(file_name, 'rb') as fp:
+            response = self.client.post("/unggah_excel/", {'excel': fp})
+        
+        messages = list(dj_messages.get_messages(response.wsgi_request))
+        msg_text = messages[0].message
+
+        self.assertIn('Kategori', msg_text)
+
+    def test_upload_excel_upload_file_success(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+
+        Category(name='Computer Science').save()
+        Category(name='Machine Learning').save()
+        Category(name='Deep Learning').save()
+
+        field_lengths = {
+            'title':50,
+            'author':30,
+            'publisher':30,
+        }
+
+        categories = ['Computer Science','Machine Learning','Deep Learning']
+        
+        file_name, data_frame = self.create_dummy_excel(field_lengths=field_lengths, categories=categories)
+
+        with open(file_name, 'rb') as fp:
+            response = self.client.post("/unggah_excel/", {'excel': fp})
+        
+        title = data_frame['Title'][0]
+        self.assertTrue(Materi.objects.get(title=title))
+
+    def test_upload_excel_download_template(self):
+        self.client.login(email="kontributor@gov.id",
+                          password="kontributor")
+
+        response = self.client.get("/unggah_excel/?template=1")
+        
+        self.assertEquals(response['Content-Type'],'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+        self.assertEquals(response['Content-Disposition'],'attachment; filename=template.xlsx')
+
+
+
 class DashboardKontributorViewTest(TestCase):
     def setUp(self):
         self.client = Client()

app/urls.py

@@ -3,7 +3,7 @@ from django.urls import path, re_path
 from app import views
 from app.views import (DashboardKontributorView, ProfilKontributorView,
                        SuksesLoginAdminView, SuksesLoginKontributorView,
-                       SuntingProfilView, UploadMateriHTML, UploadMateriView,
+                       SuntingProfilView, UploadMateriHTML, UploadMateriView, UploadMateriExcelView,
                        ProfilAdminView, PostsView, SuntingProfilAdminView,
                        ReqMateriView, KatalogPerKontributorView)
 
@@ -18,6 +18,7 @@ urlpatterns = [
     path("dashboard/", DashboardKontributorView.as_view(), name="dashboard"),
     path("revisi/materi/<int:pk>/", views.RevisiMateriView.as_view(), name="revisi"),
     path("unggah/", UploadMateriView.as_view(), name="unggah"),
+    path("unggah_excel/", UploadMateriExcelView.as_view(), name="unggah_excel"),
     path("profil/", ProfilKontributorView.as_view(), name="profil"),
     path("sunting/", SuntingProfilView.as_view(), name="sunting"),
     path("sukses-kontributor/", SuksesLoginKontributorView.as_view(),

app/views.py

@@ -20,6 +20,8 @@ from app.models import Category, Comment, Materi, Like, ViewStatistics, Download
 from app.utils.fileManagementUtil import get_random_filename, remove_image_exifdata
 from authentication.models import User
 import django
+import pandas as pd
+from io import BytesIO
 from django.contrib import messages
 
 
@@ -332,6 +334,105 @@ class UploadMateriHTML(TemplateView):
         return template_name
 
 
+class UploadMateriExcelView(TemplateView):
+    template_name = "unggah_excel.html"
+    context = {}
+
+    def get_template_names(self):
+        if self.request.path == "/unggah_excel/":
+            template_name = "unggah_excel.html"
+        return template_name
+
+    def get(self, request, *args, **kwargs):
+
+        if 'template' in self.request.GET:
+
+            data_frame = pd.DataFrame({
+                'Title': [],
+                'Author': [],
+                'Publisher': [],
+                'Categories': [],
+                'Description': [],
+            })
+
+            with BytesIO() as b:
+                writer = pd.ExcelWriter(b, engine='xlsxwriter') #pylint: disable=abstract-class-instantiated
+                data_frame.to_excel(writer, index=0)
+                writer.save()
+                response = HttpResponse(
+                        b.getvalue(), 
+                        content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+
+                response["Content-Disposition"] = "attachment; filename=template.xlsx"
+
+                return response
+
+        else:
+            context = self.get_context_data(**kwargs)
+            return self.render_to_response(context)
+
+
+    def post(self, request, *args, **kwargs):
+        excel_file = request.FILES['excel']
+        excel = pd.read_excel(excel_file)        
+
+        row,lines = excel.shape
+        categories = Category.objects.all()
+        
+        field_length = {
+            'title' : 50,
+            'author' : 30,
+            'publisher' : 30,
+        }
+
+        message = None
+
+        # First pass, validate input
+        for i in range(row):
+            
+            # Validate Categories
+            for c in excel['Categories'][i].split(","):
+                sel_cat = categories.filter(name=c)
+                if sel_cat.count() == 0:
+                    message = f"Kategori %s tidak ditemukan" % c
+                    break
+
+            if len(excel['Title'][i]) > field_length['title']:
+                message = f"Title maksimal %d karakter" % field_length['title']
+
+            if len(excel['Author'][i]) > field_length['author']:
+                message = f"Author maksimal %d karakter" % field_length['author']
+
+            if len(excel['Publisher'][i]) > field_length['publisher']:
+                message = f"Publisher maksimal %d karakter" % field_length['publisher']
+
+            if message != None:
+                break
+            
+        if message != None:
+            messages.error(request, message)
+            return HttpResponseRedirect('/unggah_excel/')
+
+        # Second pass, save data
+        with django.db.transaction.atomic():
+            for i in range(row):
+                materi = Materi(
+                            title=excel['Title'][i],
+                            author=excel['Author'][i],
+                            publisher=excel['Publisher'][i],
+                            descriptions=excel['Description'][i],
+                            uploader=request.user
+                        )
+                materi.save()
+                
+                for c in excel['Categories'][i].split(","):
+                    materi.categories.add(categories.get(name=c))
+
+        messages.success(request, 'Materi berhasil diunggah')
+
+        return HttpResponseRedirect('/unggah_excel/')
+
+
 class DashboardKontributorView(TemplateView):
     template_name = "dashboard.html"
 

requirements.txt

@@ -44,7 +44,9 @@ lazy-object-proxy==1.4.3
 mccabe==0.6.1
 more-itertools==8.2.0
 nodeenv==1.3.5
+numpy==1.19.2
 packaging==20.3
+pandas==1.1.3
 pathspec==0.8.0
 Pillow==7.1.1
 pluggy==0.13.1
@@ -82,4 +84,6 @@ virtualenv==20.0.18
 wcwidth==0.1.9
 whitenoise==5.0.1
 wrapt==1.11.2
+xlrd==1.2.0
+XlsxWriter==1.3.6
 zipp==3.1.0