feat: jira issue importer (#476)

* dev: initialize jira importer

* dev: create service import for jira

* dev: update task to create all users for project and workspace and also create assignees when importing bulk assignees

* dev: create bulk modules import endpoint for jira epics

* dev: create bulk module issues when importing modules

apiserver/plane/api/urls.py

@@ -99,6 +99,7 @@ from plane.api.views import (
     ModuleIssueViewSet,
     ModuleFavoriteViewSet,
     ModuleLinkViewSet,
+    BulkImportModulesEndpoint,
     ## End Modules
     # Pages
     PageViewSet,
@@ -904,6 +905,11 @@ urlpatterns = [
         ),
         name="user-favorite-module",
     ),
+    path(
+        "workspaces/<str:slug>/projects/<uuid:project_id>/bulk-import-modules/<str:service>/",
+        BulkImportModulesEndpoint.as_view(),
+        name="bulk-modules-create",
+    ),
     ## End Modules
     # Pages
     path(

apiserver/plane/api/views/__init__.py

@@ -108,6 +108,7 @@ from .importer import (
     ImportServiceEndpoint,
     UpdateServiceImportStatusEndpoint,
     BulkImportIssuesEndpoint,
+    BulkImportModulesEndpoint
 )
 
 from .page import PageViewSet, PageBlockViewSet, PageFavoriteViewSet, CreateIssueFromPageBlockEndpoint

apiserver/plane/api/views/importer.py

@@ -1,3 +1,6 @@
+# Python imports
+import uuid
+
 # Third party imports
 from rest_framework import status
 from rest_framework.response import Response
@@ -21,9 +24,18 @@ from plane.db.models import (
     IssueLink,
     IssueLabel,
     Workspace,
+    IssueAssignee,
+    Module,
+    ModuleLink,
+    ModuleIssue,
+)
+from plane.api.serializers import (
+    ImporterSerializer,
+    IssueFlatSerializer,
+    ModuleSerializer,
 )
-from plane.api.serializers import ImporterSerializer, IssueFlatSerializer
 from plane.utils.integrations.github import get_github_repo_details
+from plane.utils.importers.jira import jira_project_issue_summary
 from plane.bgtasks.importer_task import service_importer
 from plane.utils.html_processor import strip_tags
 
@@ -52,6 +64,30 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView):
                     status=status.HTTP_200_OK,
                 )
 
+            if service == "jira":
+                project_name = request.data.get("project_name", "")
+                api_token = request.data.get("api_token", "")
+                email = request.data.get("email", "")
+                cloud_hostname = request.data.get("cloud_hostname", "")
+                if (
+                    not bool(project_name)
+                    or not bool(api_token)
+                    or not bool(email)
+                    or not bool(cloud_hostname)
+                ):
+                    return Response(
+                        {
+                            "error": "Project name, Project key, API token, Cloud hostname and email are requied"
+                        },
+                        status=status.HTTP_400_BAD_REQUEST,
+                    )
+
+                return Response(
+                    jira_project_issue_summary(
+                        email, api_token, project_name, cloud_hostname
+                    ),
+                    status=status.HTTP_200_OK,
+                )
             return Response(
                 {"error": "Service not supported yet"},
                 status=status.HTTP_400_BAD_REQUEST,
@@ -61,6 +97,113 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView):
                 {"error": "Requested integration was not installed in the workspace"},
                 status=status.HTTP_400_BAD_REQUEST,
             )
+        except Exception as e:
+            print(e)
+            return Response(
+                {"error": "Something went wrong please try again later"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+
+class ImportServiceEndpoint(BaseAPIView):
+    def post(self, request, slug, service):
+        try:
+            project_id = request.data.get("project_id", False)
+
+            if not project_id:
+                return Response(
+                    {"error": "Project ID is required"},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            workspace = Workspace.objects.get(slug=slug)
+
+            if service == "github":
+                data = request.data.get("data", False)
+                metadata = request.data.get("metadata", False)
+                config = request.data.get("config", False)
+                if not data or not metadata or not config:
+                    return Response(
+                        {"error": "Data, config and metadata are required"},
+                        status=status.HTTP_400_BAD_REQUEST,
+                    )
+
+                api_token = APIToken.objects.filter(
+                    user=request.user, workspace=workspace
+                ).first()
+                if api_token is None:
+                    api_token = APIToken.objects.create(
+                        user=request.user,
+                        label="Importer",
+                        workspace=workspace,
+                    )
+
+                importer = Importer.objects.create(
+                    service=service,
+                    project_id=project_id,
+                    status="queued",
+                    initiated_by=request.user,
+                    data=data,
+                    metadata=metadata,
+                    token=api_token,
+                    config=config,
+                    created_by=request.user,
+                    updated_by=request.user,
+                )
+
+                service_importer.delay(service, importer.id)
+                serializer = ImporterSerializer(importer)
+                return Response(serializer.data, status=status.HTTP_201_CREATED)
+
+            if service == "jira":
+                data = request.data.get("data", False)
+                metadata = request.data.get("metadata", False)
+                config = request.data.get("config", False)
+                if not data or not metadata:
+                    return Response(
+                        {"error": "Data, config and metadata are required"},
+                        status=status.HTTP_400_BAD_REQUEST,
+                    )
+                api_token = APIToken.objects.filter(
+                    user=request.user, workspace=workspace
+                ).first()
+                if api_token is None:
+                    api_token = APIToken.objects.create(
+                        user=request.user,
+                        label="Importer",
+                        workspace=workspace,
+                    )
+
+                importer = Importer.objects.create(
+                    service=service,
+                    project_id=project_id,
+                    status="queued",
+                    initiated_by=request.user,
+                    data=data,
+                    metadata=metadata,
+                    token=api_token,
+                    config=config,
+                    created_by=request.user,
+                    updated_by=request.user,
+                )
+
+                service_importer.delay(service, importer.id)
+                serializer = ImporterSerializer(importer)
+                return Response(serializer.data, status=status.HTTP_201_CREATED)
+
+            return Response(
+                {"error": "Servivce not supported yet"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        except (
+            Workspace.DoesNotExist,
+            WorkspaceIntegration.DoesNotExist,
+            Project.DoesNotExist,
+        ) as e:
+            return Response(
+                {"error": "Workspace Integration or Project does not exist"},
+                status=status.HTTP_404_NOT_FOUND,
+            )
         except Exception as e:
             capture_exception(e)
             return Response(
@@ -68,6 +211,36 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView):
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
+    def get(self, request, slug):
+        try:
+            imports = Importer.objects.filter(workspace__slug=slug)
+            serializer = ImporterSerializer(imports, many=True)
+            return Response(serializer.data)
+        except Exception as e:
+            capture_exception(e)
+            return Response(
+                {"error": "Something went wrong please try again later"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+
+class UpdateServiceImportStatusEndpoint(BaseAPIView):
+    def post(self, request, slug, project_id, service, importer_id):
+        try:
+            importer = Importer.objects.get(
+                pk=importer_id,
+                workspace__slug=slug,
+                project_id=project_id,
+                service=service,
+            )
+            importer.status = request.data.get("status", "processing")
+            importer.save()
+            return Response(status.HTTP_200_OK)
+        except Importer.DoesNotExist:
+            return Response(
+                {"error": "Importer does not exist"}, status=status.HTTP_404_NOT_FOUND
+            )
+
 
 class BulkImportIssuesEndpoint(BaseAPIView):
     def post(self, request, slug, project_id, service):
@@ -115,7 +288,9 @@ class BulkImportIssuesEndpoint(BaseAPIView):
                     Issue(
                         project_id=project_id,
                         workspace_id=project.workspace_id,
-                        state=default_state,
+                        state_id=issue_data.get("state")
+                        if issue_data.get("state", False)
+                        else default_state.id,
                         name=issue_data.get("name", "Issue Created through Bulk"),
                         description_html=issue_data.get("description_html", "<p></p>"),
                         description_stripped=(
@@ -130,6 +305,7 @@ class BulkImportIssuesEndpoint(BaseAPIView):
                         sort_order=largest_sort_order,
                         start_date=issue_data.get("start_date", None),
                         target_date=issue_data.get("target_date", None),
+                        priority=issue_data.get("priority", None),
                     )
                 )
 
@@ -172,7 +348,29 @@ class BulkImportIssuesEndpoint(BaseAPIView):
                     for label_id in labels_list
                 ]
 
-            _ = IssueLabel.objects.bulk_create(bulk_issue_labels, batch_size=100)
+            _ = IssueLabel.objects.bulk_create(
+                bulk_issue_labels, batch_size=100, ignore_conflicts=True
+            )
+
+            # Attach Assignees
+            bulk_issue_assignees = []
+            for issue, issue_data in zip(issues, issues_data):
+                assignees_list = issue_data.get("assignees_list", [])
+                bulk_issue_assignees = bulk_issue_assignees + [
+                    IssueAssignee(
+                        issue=issue,
+                        assignee_id=assignee_id,
+                        project_id=project_id,
+                        workspace_id=project.workspace_id,
+                        created_by=request.user,
+                        updated_by=request.user,
+                    )
+                    for assignee_id in assignees_list
+                ]
+
+            _ = IssueAssignee.objects.bulk_create(
+                bulk_issue_assignees, batch_size=100, ignore_conflicts=True
+            )
 
             # Track the issue activities
             IssueActivity.objects.bulk_create(
@@ -241,62 +439,75 @@ class BulkImportIssuesEndpoint(BaseAPIView):
             )
 
 
-class ImportServiceEndpoint(BaseAPIView):
+class BulkImportModulesEndpoint(BaseAPIView):
-    def post(self, request, slug, service):
+    def post(self, request, slug, project_id, service):
         try:
-            project_id = request.data.get("project_id", False)
+            modules_data = request.data.get("modules_data", [])
+            project = Project.objects.get(pk=project_id, workspace__slug=slug)
 
-            if not project_id:
+            modules = Module.objects.bulk_create(
-                return Response(
+                [
-                    {"error": "Project ID is required"},
+                    Module(
-                    status=status.HTTP_400_BAD_REQUEST,
+                        name=module.get("name", uuid.uuid4().hex),
-                )
+                        description=module.get("description", ""),
-
+                        start_date=module.get("start_date", None),
-            workspace = Workspace.objects.get(slug=slug)
+                        target_date=module.get("target_date", None),
-
+                        project_id=project_id,
-            if service == "github":
+                        workspace_id=project.workspace_id,
-                data = request.data.get("data", False)
+                        created_by=request.user,
-                metadata = request.data.get("metadata", False)
+                        updated_by=request.user,
-                config = request.data.get("config", False)
-                if not data or not metadata or not config:
-                    return Response(
-                        {"error": "Data, config and metadata are required"},
-                        status=status.HTTP_400_BAD_REQUEST,
                     )
-
+                    for module in modules_data
-                api_token = APIToken.objects.filter(user=request.user).first()
+                ],
-                if api_token is None:
+                batch_size=100,
-                    api_token = APIToken.objects.create(
+                ignore_conflicts=True,
-                        user=request.user,
-                        label="Importer",
-                        workspace=workspace,
-                    )
-
-                importer = Importer.objects.create(
-                    service=service,
-                    project_id=project_id,
-                    status="queued",
-                    initiated_by=request.user,
-                    data=data,
-                    metadata=metadata,
-                    token=api_token,
-                    config=config,
-                    created_by=request.user,
-                    updated_by=request.user,
-                )
-
-                service_importer.delay(service, importer.id)
-                serializer = ImporterSerializer(importer)
-                return Response(serializer.data, status=status.HTTP_200_OK)
-
-            return Response(
-                {"error": "Servivce not supported yet"},
-                status=status.HTTP_400_BAD_REQUEST,
             )
-        except (Workspace.DoesNotExist, WorkspaceIntegration.DoesNotExist) as e:
+
+            _ = ModuleLink.objects.bulk_create(
+                [
+                    ModuleLink(
+                        module=module,
+                        url=module_data.get("link", {}).get("url", "https://plane.so"),
+                        title=module_data.get("link", {}).get(
+                            "title", "Original Issue"
+                        ),
+                        project_id=project_id,
+                        workspace_id=project.workspace_id,
+                        created_by=request.user,
+                        updated_by=request.user,
+                    )
+                    for module, module_data in zip(modules, modules_data)
+                ],
+                batch_size=100,
+                ignore_conflicts=True,
+            )
+
+            bulk_module_issues = []
+            for module, module_data in zip(modules, modules_data):
+                module_issues_list = module_data.get("module_issues_list", [])
+                bulk_module_issues = bulk_module_issues + [
+                    ModuleIssue(
+                        issue_id=issue,
+                        module=module,
+                        project_id=project_id,
+                        workspace_id=project.workspace_id,
+                        created_by=request.user,
+                        updated_by=request.user,
+                    )
+                    for issue in module_issues_list
+                ]
+
+            _ = ModuleIssue.objects.bulk_create(
+                bulk_module_issues, batch_size=100, ignore_conflicts=True
+            )
+
+            serializer = ModuleSerializer(modules, many=True)
             return Response(
-                {"error": "Workspace Integration does not exist"},
+                {"modules": serializer.data}, status=status.HTTP_201_CREATED
-                status=status.HTTP_404_NOT_FOUND,
+            )
+        except Project.DoesNotExist:
+            return Response(
+                {"error": "Project does not exist"}, status=status.HTTP_404_NOT_FOUND
             )
         except Exception as e:
             capture_exception(e)
@@ -304,33 +515,3 @@ class ImportServiceEndpoint(BaseAPIView):
                 {"error": "Something went wrong please try again later"},
                 status=status.HTTP_400_BAD_REQUEST,
             )
-
-    def get(self, request, slug):
-        try:
-            imports = Importer.objects.filter(workspace__slug=slug)
-            serializer = ImporterSerializer(imports, many=True)
-            return Response(serializer.data)
-        except Exception as e:
-            capture_exception(e)
-            return Response(
-                {"error": "Something went wrong please try again later"},
-                status=status.HTTP_400_BAD_REQUEST,
-            )
-
-
-class UpdateServiceImportStatusEndpoint(BaseAPIView):
-    def post(self, request, slug, project_id, service, importer_id):
-        try:
-            importer = Importer.objects.get(
-                pk=importer_id,
-                workspace__slug=slug,
-                project_id=project_id,
-                service=service,
-            )
-            importer.status = request.data.get("status", "processing")
-            importer.save()
-            return Response(status.HTTP_200_OK)
-        except Importer.DoesNotExist:
-            return Response(
-                {"error": "Importer does not exist"}, status=status.HTTP_404_NOT_FOUND
-            )

apiserver/plane/bgtasks/importer_task.py

@@ -51,17 +51,29 @@ def service_importer(service, importer_id):
                 if user.get("import", False) == "invite"
             ],
             batch_size=10,
+            ignore_conflicts=True,
         )
 
+        workspace_users = User.objects.filter(
+            email__in=[
+                user.get("email").strip().lower()
+                for user in users
+                if user.get("import", False) == "invite"
+                or user.get("import", False) == "map"
+            ]
+        )
+
+
         # Add new users to Workspace and project automatically
         WorkspaceMember.objects.bulk_create(
             [
                 WorkspaceMember(member=user, workspace_id=importer.workspace_id)
-                for user in new_users
+                for user in workspace_users
             ],
             batch_size=100,
             ignore_conflicts=True,
         )
+
         ProjectMember.objects.bulk_create(
             [
                 ProjectMember(
@@ -69,7 +81,7 @@ def service_importer(service, importer_id):
                     workspace_id=importer.workspace_id,
                     member=user,
                 )
-                for user in new_users
+                for user in workspace_users
             ],
             batch_size=100,
             ignore_conflicts=True,

apiserver/plane/db/models/importer.py

@@ -7,7 +7,13 @@ from . import ProjectBaseModel
 
 
 class Importer(ProjectBaseModel):
-    service = models.CharField(max_length=50, choices=(("github", "GitHub"),))
+    service = models.CharField(
+        max_length=50,
+        choices=(
+            ("github", "GitHub"),
+            ("jira", "Jira"),
+        ),
+    )
     status = models.CharField(
         max_length=50,
         choices=(

apiserver/plane/utils/importers/jira.py

@@ -0,0 +1,53 @@
+import requests
+from requests.auth import HTTPBasicAuth
+from sentry_sdk import capture_exception
+
+
+def jira_project_issue_summary(email, api_token, project_name, hostname):
+    try:
+        auth = HTTPBasicAuth(email, api_token)
+        headers = {"Accept": "application/json"}
+
+        issue_url = f"https://{hostname}/rest/api/3/search?jql=project={project_name} AND issuetype=Story"
+        issue_response = requests.request(
+            "GET", issue_url, headers=headers, auth=auth
+        ).json()["total"]
+
+        module_url = f"https://{hostname}/rest/api/3/search?jql=project={project_name} AND issuetype=Epic"
+        module_response = requests.request(
+            "GET", module_url, headers=headers, auth=auth
+        ).json()["total"]
+
+        status_url = f"https://{hostname}/rest/api/3/status/?jql=project={project_name}"
+        status_response = requests.request(
+            "GET", status_url, headers=headers, auth=auth
+        ).json()
+
+        labels_url = f"https://{hostname}/rest/api/3/label/?jql=project={project_name}"
+        labels_response = requests.request(
+            "GET", labels_url, headers=headers, auth=auth
+        ).json()["total"]
+
+        users_url = (
+            f"https://{hostname}/rest/api/3/users/search?jql=project={project_name}"
+        )
+        users_response = requests.request(
+            "GET", users_url, headers=headers, auth=auth
+        ).json()
+
+        return {
+            "issues": issue_response,
+            "modules": module_response,
+            "labels": labels_response,
+            "states": len(status_response),
+            "users": (
+                [
+                    user
+                    for user in users_response
+                    if user.get("accountType") == "atlassian"
+                ]
+            ),
+        }
+    except Exception as e:
+        capture_exception(e)
+        return {"error": "Something went wrong could not fetch information from jira"}