From 5e81600e3817ee5af60d0bee6ef2aa882566521e Mon Sep 17 00:00:00 2001 From: pablohashescobar <118773738+pablohashescobar@users.noreply.github.com> Date: Wed, 22 Mar 2023 01:36:38 +0530 Subject: [PATCH] feat: jira issue importer (#476) * dev: initialize jira importer * dev: create service import for jira * dev: update task to create all users for project and workspace and also create assignees when importing bulk assignees * dev: create bulk modules import endpoint for jira epics * dev: create bulk module issues when importing modules --- apiserver/plane/api/urls.py | 6 + apiserver/plane/api/views/__init__.py | 1 + apiserver/plane/api/views/importer.py | 349 +++++++++++++++----- apiserver/plane/bgtasks/importer_task.py | 16 +- apiserver/plane/db/models/importer.py | 8 +- apiserver/plane/utils/importers/__init__.py | 0 apiserver/plane/utils/importers/jira.py | 53 +++ 7 files changed, 346 insertions(+), 87 deletions(-) create mode 100644 apiserver/plane/utils/importers/__init__.py create mode 100644 apiserver/plane/utils/importers/jira.py diff --git a/apiserver/plane/api/urls.py b/apiserver/plane/api/urls.py index 26e94d477..bf713a927 100644 --- a/apiserver/plane/api/urls.py +++ b/apiserver/plane/api/urls.py @@ -99,6 +99,7 @@ from plane.api.views import ( ModuleIssueViewSet, ModuleFavoriteViewSet, ModuleLinkViewSet, + BulkImportModulesEndpoint, ## End Modules # Pages PageViewSet, @@ -904,6 +905,11 @@ urlpatterns = [ ), name="user-favorite-module", ), + path( + "workspaces//projects//bulk-import-modules//", + BulkImportModulesEndpoint.as_view(), + name="bulk-modules-create", + ), ## End Modules # Pages path( diff --git a/apiserver/plane/api/views/__init__.py b/apiserver/plane/api/views/__init__.py index 0e99069b5..f5dfb2bac 100644 --- a/apiserver/plane/api/views/__init__.py +++ b/apiserver/plane/api/views/__init__.py @@ -108,6 +108,7 @@ from .importer import ( ImportServiceEndpoint, UpdateServiceImportStatusEndpoint, BulkImportIssuesEndpoint, + BulkImportModulesEndpoint ) from .page import PageViewSet, PageBlockViewSet, PageFavoriteViewSet, CreateIssueFromPageBlockEndpoint diff --git a/apiserver/plane/api/views/importer.py b/apiserver/plane/api/views/importer.py index 6d7d11a13..7d2bc14b4 100644 --- a/apiserver/plane/api/views/importer.py +++ b/apiserver/plane/api/views/importer.py @@ -1,3 +1,6 @@ +# Python imports +import uuid + # Third party imports from rest_framework import status from rest_framework.response import Response @@ -21,9 +24,18 @@ from plane.db.models import ( IssueLink, IssueLabel, Workspace, + IssueAssignee, + Module, + ModuleLink, + ModuleIssue, +) +from plane.api.serializers import ( + ImporterSerializer, + IssueFlatSerializer, + ModuleSerializer, ) -from plane.api.serializers import ImporterSerializer, IssueFlatSerializer from plane.utils.integrations.github import get_github_repo_details +from plane.utils.importers.jira import jira_project_issue_summary from plane.bgtasks.importer_task import service_importer from plane.utils.html_processor import strip_tags @@ -52,6 +64,30 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView): status=status.HTTP_200_OK, ) + if service == "jira": + project_name = request.data.get("project_name", "") + api_token = request.data.get("api_token", "") + email = request.data.get("email", "") + cloud_hostname = request.data.get("cloud_hostname", "") + if ( + not bool(project_name) + or not bool(api_token) + or not bool(email) + or not bool(cloud_hostname) + ): + return Response( + { + "error": "Project name, Project key, API token, Cloud hostname and email are requied" + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + return Response( + jira_project_issue_summary( + email, api_token, project_name, cloud_hostname + ), + status=status.HTTP_200_OK, + ) return Response( {"error": "Service not supported yet"}, status=status.HTTP_400_BAD_REQUEST, @@ -61,6 +97,113 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView): {"error": "Requested integration was not installed in the workspace"}, status=status.HTTP_400_BAD_REQUEST, ) + except Exception as e: + print(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + +class ImportServiceEndpoint(BaseAPIView): + def post(self, request, slug, service): + try: + project_id = request.data.get("project_id", False) + + if not project_id: + return Response( + {"error": "Project ID is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + workspace = Workspace.objects.get(slug=slug) + + if service == "github": + data = request.data.get("data", False) + metadata = request.data.get("metadata", False) + config = request.data.get("config", False) + if not data or not metadata or not config: + return Response( + {"error": "Data, config and metadata are required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + api_token = APIToken.objects.filter( + user=request.user, workspace=workspace + ).first() + if api_token is None: + api_token = APIToken.objects.create( + user=request.user, + label="Importer", + workspace=workspace, + ) + + importer = Importer.objects.create( + service=service, + project_id=project_id, + status="queued", + initiated_by=request.user, + data=data, + metadata=metadata, + token=api_token, + config=config, + created_by=request.user, + updated_by=request.user, + ) + + service_importer.delay(service, importer.id) + serializer = ImporterSerializer(importer) + return Response(serializer.data, status=status.HTTP_201_CREATED) + + if service == "jira": + data = request.data.get("data", False) + metadata = request.data.get("metadata", False) + config = request.data.get("config", False) + if not data or not metadata: + return Response( + {"error": "Data, config and metadata are required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + api_token = APIToken.objects.filter( + user=request.user, workspace=workspace + ).first() + if api_token is None: + api_token = APIToken.objects.create( + user=request.user, + label="Importer", + workspace=workspace, + ) + + importer = Importer.objects.create( + service=service, + project_id=project_id, + status="queued", + initiated_by=request.user, + data=data, + metadata=metadata, + token=api_token, + config=config, + created_by=request.user, + updated_by=request.user, + ) + + service_importer.delay(service, importer.id) + serializer = ImporterSerializer(importer) + return Response(serializer.data, status=status.HTTP_201_CREATED) + + return Response( + {"error": "Servivce not supported yet"}, + status=status.HTTP_400_BAD_REQUEST, + ) + except ( + Workspace.DoesNotExist, + WorkspaceIntegration.DoesNotExist, + Project.DoesNotExist, + ) as e: + return Response( + {"error": "Workspace Integration or Project does not exist"}, + status=status.HTTP_404_NOT_FOUND, + ) except Exception as e: capture_exception(e) return Response( @@ -68,6 +211,36 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView): status=status.HTTP_400_BAD_REQUEST, ) + def get(self, request, slug): + try: + imports = Importer.objects.filter(workspace__slug=slug) + serializer = ImporterSerializer(imports, many=True) + return Response(serializer.data) + except Exception as e: + capture_exception(e) + return Response( + {"error": "Something went wrong please try again later"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + +class UpdateServiceImportStatusEndpoint(BaseAPIView): + def post(self, request, slug, project_id, service, importer_id): + try: + importer = Importer.objects.get( + pk=importer_id, + workspace__slug=slug, + project_id=project_id, + service=service, + ) + importer.status = request.data.get("status", "processing") + importer.save() + return Response(status.HTTP_200_OK) + except Importer.DoesNotExist: + return Response( + {"error": "Importer does not exist"}, status=status.HTTP_404_NOT_FOUND + ) + class BulkImportIssuesEndpoint(BaseAPIView): def post(self, request, slug, project_id, service): @@ -115,7 +288,9 @@ class BulkImportIssuesEndpoint(BaseAPIView): Issue( project_id=project_id, workspace_id=project.workspace_id, - state=default_state, + state_id=issue_data.get("state") + if issue_data.get("state", False) + else default_state.id, name=issue_data.get("name", "Issue Created through Bulk"), description_html=issue_data.get("description_html", "

"), description_stripped=( @@ -130,6 +305,7 @@ class BulkImportIssuesEndpoint(BaseAPIView): sort_order=largest_sort_order, start_date=issue_data.get("start_date", None), target_date=issue_data.get("target_date", None), + priority=issue_data.get("priority", None), ) ) @@ -172,7 +348,29 @@ class BulkImportIssuesEndpoint(BaseAPIView): for label_id in labels_list ] - _ = IssueLabel.objects.bulk_create(bulk_issue_labels, batch_size=100) + _ = IssueLabel.objects.bulk_create( + bulk_issue_labels, batch_size=100, ignore_conflicts=True + ) + + # Attach Assignees + bulk_issue_assignees = [] + for issue, issue_data in zip(issues, issues_data): + assignees_list = issue_data.get("assignees_list", []) + bulk_issue_assignees = bulk_issue_assignees + [ + IssueAssignee( + issue=issue, + assignee_id=assignee_id, + project_id=project_id, + workspace_id=project.workspace_id, + created_by=request.user, + updated_by=request.user, + ) + for assignee_id in assignees_list + ] + + _ = IssueAssignee.objects.bulk_create( + bulk_issue_assignees, batch_size=100, ignore_conflicts=True + ) # Track the issue activities IssueActivity.objects.bulk_create( @@ -241,62 +439,75 @@ class BulkImportIssuesEndpoint(BaseAPIView): ) -class ImportServiceEndpoint(BaseAPIView): - def post(self, request, slug, service): +class BulkImportModulesEndpoint(BaseAPIView): + def post(self, request, slug, project_id, service): try: - project_id = request.data.get("project_id", False) + modules_data = request.data.get("modules_data", []) + project = Project.objects.get(pk=project_id, workspace__slug=slug) - if not project_id: - return Response( - {"error": "Project ID is required"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - workspace = Workspace.objects.get(slug=slug) - - if service == "github": - data = request.data.get("data", False) - metadata = request.data.get("metadata", False) - config = request.data.get("config", False) - if not data or not metadata or not config: - return Response( - {"error": "Data, config and metadata are required"}, - status=status.HTTP_400_BAD_REQUEST, + modules = Module.objects.bulk_create( + [ + Module( + name=module.get("name", uuid.uuid4().hex), + description=module.get("description", ""), + start_date=module.get("start_date", None), + target_date=module.get("target_date", None), + project_id=project_id, + workspace_id=project.workspace_id, + created_by=request.user, + updated_by=request.user, ) - - api_token = APIToken.objects.filter(user=request.user).first() - if api_token is None: - api_token = APIToken.objects.create( - user=request.user, - label="Importer", - workspace=workspace, - ) - - importer = Importer.objects.create( - service=service, - project_id=project_id, - status="queued", - initiated_by=request.user, - data=data, - metadata=metadata, - token=api_token, - config=config, - created_by=request.user, - updated_by=request.user, - ) - - service_importer.delay(service, importer.id) - serializer = ImporterSerializer(importer) - return Response(serializer.data, status=status.HTTP_200_OK) - - return Response( - {"error": "Servivce not supported yet"}, - status=status.HTTP_400_BAD_REQUEST, + for module in modules_data + ], + batch_size=100, + ignore_conflicts=True, ) - except (Workspace.DoesNotExist, WorkspaceIntegration.DoesNotExist) as e: + + _ = ModuleLink.objects.bulk_create( + [ + ModuleLink( + module=module, + url=module_data.get("link", {}).get("url", "https://plane.so"), + title=module_data.get("link", {}).get( + "title", "Original Issue" + ), + project_id=project_id, + workspace_id=project.workspace_id, + created_by=request.user, + updated_by=request.user, + ) + for module, module_data in zip(modules, modules_data) + ], + batch_size=100, + ignore_conflicts=True, + ) + + bulk_module_issues = [] + for module, module_data in zip(modules, modules_data): + module_issues_list = module_data.get("module_issues_list", []) + bulk_module_issues = bulk_module_issues + [ + ModuleIssue( + issue_id=issue, + module=module, + project_id=project_id, + workspace_id=project.workspace_id, + created_by=request.user, + updated_by=request.user, + ) + for issue in module_issues_list + ] + + _ = ModuleIssue.objects.bulk_create( + bulk_module_issues, batch_size=100, ignore_conflicts=True + ) + + serializer = ModuleSerializer(modules, many=True) return Response( - {"error": "Workspace Integration does not exist"}, - status=status.HTTP_404_NOT_FOUND, + {"modules": serializer.data}, status=status.HTTP_201_CREATED + ) + except Project.DoesNotExist: + return Response( + {"error": "Project does not exist"}, status=status.HTTP_404_NOT_FOUND ) except Exception as e: capture_exception(e) @@ -304,33 +515,3 @@ class ImportServiceEndpoint(BaseAPIView): {"error": "Something went wrong please try again later"}, status=status.HTTP_400_BAD_REQUEST, ) - - def get(self, request, slug): - try: - imports = Importer.objects.filter(workspace__slug=slug) - serializer = ImporterSerializer(imports, many=True) - return Response(serializer.data) - except Exception as e: - capture_exception(e) - return Response( - {"error": "Something went wrong please try again later"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - -class UpdateServiceImportStatusEndpoint(BaseAPIView): - def post(self, request, slug, project_id, service, importer_id): - try: - importer = Importer.objects.get( - pk=importer_id, - workspace__slug=slug, - project_id=project_id, - service=service, - ) - importer.status = request.data.get("status", "processing") - importer.save() - return Response(status.HTTP_200_OK) - except Importer.DoesNotExist: - return Response( - {"error": "Importer does not exist"}, status=status.HTTP_404_NOT_FOUND - ) diff --git a/apiserver/plane/bgtasks/importer_task.py b/apiserver/plane/bgtasks/importer_task.py index 82050e950..f5dadf322 100644 --- a/apiserver/plane/bgtasks/importer_task.py +++ b/apiserver/plane/bgtasks/importer_task.py @@ -51,17 +51,29 @@ def service_importer(service, importer_id): if user.get("import", False) == "invite" ], batch_size=10, + ignore_conflicts=True, ) + workspace_users = User.objects.filter( + email__in=[ + user.get("email").strip().lower() + for user in users + if user.get("import", False) == "invite" + or user.get("import", False) == "map" + ] + ) + + # Add new users to Workspace and project automatically WorkspaceMember.objects.bulk_create( [ WorkspaceMember(member=user, workspace_id=importer.workspace_id) - for user in new_users + for user in workspace_users ], batch_size=100, ignore_conflicts=True, ) + ProjectMember.objects.bulk_create( [ ProjectMember( @@ -69,7 +81,7 @@ def service_importer(service, importer_id): workspace_id=importer.workspace_id, member=user, ) - for user in new_users + for user in workspace_users ], batch_size=100, ignore_conflicts=True, diff --git a/apiserver/plane/db/models/importer.py b/apiserver/plane/db/models/importer.py index d3f55b750..a61aae48c 100644 --- a/apiserver/plane/db/models/importer.py +++ b/apiserver/plane/db/models/importer.py @@ -7,7 +7,13 @@ from . import ProjectBaseModel class Importer(ProjectBaseModel): - service = models.CharField(max_length=50, choices=(("github", "GitHub"),)) + service = models.CharField( + max_length=50, + choices=( + ("github", "GitHub"), + ("jira", "Jira"), + ), + ) status = models.CharField( max_length=50, choices=( diff --git a/apiserver/plane/utils/importers/__init__.py b/apiserver/plane/utils/importers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apiserver/plane/utils/importers/jira.py b/apiserver/plane/utils/importers/jira.py new file mode 100644 index 000000000..a5888e2ec --- /dev/null +++ b/apiserver/plane/utils/importers/jira.py @@ -0,0 +1,53 @@ +import requests +from requests.auth import HTTPBasicAuth +from sentry_sdk import capture_exception + + +def jira_project_issue_summary(email, api_token, project_name, hostname): + try: + auth = HTTPBasicAuth(email, api_token) + headers = {"Accept": "application/json"} + + issue_url = f"https://{hostname}/rest/api/3/search?jql=project={project_name} AND issuetype=Story" + issue_response = requests.request( + "GET", issue_url, headers=headers, auth=auth + ).json()["total"] + + module_url = f"https://{hostname}/rest/api/3/search?jql=project={project_name} AND issuetype=Epic" + module_response = requests.request( + "GET", module_url, headers=headers, auth=auth + ).json()["total"] + + status_url = f"https://{hostname}/rest/api/3/status/?jql=project={project_name}" + status_response = requests.request( + "GET", status_url, headers=headers, auth=auth + ).json() + + labels_url = f"https://{hostname}/rest/api/3/label/?jql=project={project_name}" + labels_response = requests.request( + "GET", labels_url, headers=headers, auth=auth + ).json()["total"] + + users_url = ( + f"https://{hostname}/rest/api/3/users/search?jql=project={project_name}" + ) + users_response = requests.request( + "GET", users_url, headers=headers, auth=auth + ).json() + + return { + "issues": issue_response, + "modules": module_response, + "labels": labels_response, + "states": len(status_response), + "users": ( + [ + user + for user in users_response + if user.get("accountType") == "atlassian" + ] + ), + } + except Exception as e: + capture_exception(e) + return {"error": "Something went wrong could not fetch information from jira"}