thunderstrike/plane
thunderstrike/plane
2
0
Fork 0
forked from github/plane
Code Pull Requests Activity

feat: jira issue importer (#476)

Browse Source 
* dev: initialize jira importer

* dev: create service import for jira

* dev: update task to create all users for project and workspace and also create assignees when importing bulk assignees

* dev: create bulk modules import endpoint for jira epics

* dev: create bulk module issues when importing modules
This commit is contained in:
pablohashescobar 2023-03-22 01:36:38 +05:30 committed by GitHub
parent 846e73e3b8
commit 5e81600e38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 346 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apiserver/plane/api/urls.py
View File

@ -99,6 +99,7 @@ from plane.api.views import (
ModuleIssueViewSet,
ModuleFavoriteViewSet,
ModuleLinkViewSet,
BulkImportModulesEndpoint,
## End Modules
# Pages
PageViewSet,
@ -904,6 +905,11 @@ urlpatterns = [
),
name="user-favorite-module",
),
path(
"workspaces/<str:slug>/projects/<uuid:project_id>/bulk-import-modules/<str:service>/",
BulkImportModulesEndpoint.as_view(),
name="bulk-modules-create",
),
## End Modules
# Pages
path(

1
apiserver/plane/api/views/__init__.py
View File

@ -108,6 +108,7 @@ from .importer import (
ImportServiceEndpoint,
UpdateServiceImportStatusEndpoint,
BulkImportIssuesEndpoint,
BulkImportModulesEndpoint
)
from .page import PageViewSet, PageBlockViewSet, PageFavoriteViewSet, CreateIssueFromPageBlockEndpoint

349
apiserver/plane/api/views/importer.py
View File

@ -1,3 +1,6 @@
# Python imports
import uuid
# Third party imports
from rest_framework import status
from rest_framework.response import Response
@ -21,9 +24,18 @@ from plane.db.models import (
IssueLink,
IssueLabel,
Workspace,
IssueAssignee,
Module,
ModuleLink,
ModuleIssue,
)
from plane.api.serializers import (
ImporterSerializer,
IssueFlatSerializer,
ModuleSerializer,
)
from plane.api.serializers import ImporterSerializer, IssueFlatSerializer
from plane.utils.integrations.github import get_github_repo_details
from plane.utils.importers.jira import jira_project_issue_summary
from plane.bgtasks.importer_task import service_importer
from plane.utils.html_processor import strip_tags
@ -52,6 +64,30 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView):
status=status.HTTP_200_OK,
)
if service == "jira":
project_name = request.data.get("project_name", "")
api_token = request.data.get("api_token", "")
email = request.data.get("email", "")
cloud_hostname = request.data.get("cloud_hostname", "")
if (
not bool(project_name)
or not bool(api_token)
or not bool(email)
or not bool(cloud_hostname)
):
return Response(
{
"error": "Project name, Project key, API token, Cloud hostname and email are requied"
},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(
jira_project_issue_summary(
email, api_token, project_name, cloud_hostname
),
status=status.HTTP_200_OK,
)
return Response(
{"error": "Service not supported yet"},
status=status.HTTP_400_BAD_REQUEST,
@ -61,6 +97,113 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView):
{"error": "Requested integration was not installed in the workspace"},
status=status.HTTP_400_BAD_REQUEST,
)
except Exception as e:
print(e)
return Response(
{"error": "Something went wrong please try again later"},
status=status.HTTP_400_BAD_REQUEST,
)
class ImportServiceEndpoint(BaseAPIView):
def post(self, request, slug, service):
try:
project_id = request.data.get("project_id", False)
if not project_id:
return Response(
{"error": "Project ID is required"},
status=status.HTTP_400_BAD_REQUEST,
)
workspace = Workspace.objects.get(slug=slug)
if service == "github":
data = request.data.get("data", False)
metadata = request.data.get("metadata", False)
config = request.data.get("config", False)
if not data or not metadata or not config:
return Response(
{"error": "Data, config and metadata are required"},
status=status.HTTP_400_BAD_REQUEST,
)
api_token = APIToken.objects.filter(
user=request.user, workspace=workspace
).first()
if api_token is None:
api_token = APIToken.objects.create(
user=request.user,
label="Importer",
workspace=workspace,
)
importer = Importer.objects.create(
service=service,
project_id=project_id,
status="queued",
initiated_by=request.user,
data=data,
metadata=metadata,
token=api_token,
config=config,
created_by=request.user,
updated_by=request.user,
)
service_importer.delay(service, importer.id)
serializer = ImporterSerializer(importer)
return Response(serializer.data, status=status.HTTP_201_CREATED)
if service == "jira":
data = request.data.get("data", False)
metadata = request.data.get("metadata", False)
config = request.data.get("config", False)
if not data or not metadata:
return Response(
{"error": "Data, config and metadata are required"},
status=status.HTTP_400_BAD_REQUEST,
)
api_token = APIToken.objects.filter(
user=request.user, workspace=workspace
).first()
if api_token is None:
api_token = APIToken.objects.create(
user=request.user,
label="Importer",
workspace=workspace,
)
importer = Importer.objects.create(
service=service,
project_id=project_id,
status="queued",
initiated_by=request.user,
data=data,
metadata=metadata,
token=api_token,
config=config,
created_by=request.user,
updated_by=request.user,
)
service_importer.delay(service, importer.id)
serializer = ImporterSerializer(importer)
return Response(serializer.data, status=status.HTTP_201_CREATED)
return Response(
{"error": "Servivce not supported yet"},
status=status.HTTP_400_BAD_REQUEST,
)
except (
Workspace.DoesNotExist,
WorkspaceIntegration.DoesNotExist,
Project.DoesNotExist,
) as e:
return Response(
{"error": "Workspace Integration or Project does not exist"},
status=status.HTTP_404_NOT_FOUND,
)
except Exception as e:
capture_exception(e)
return Response(
@ -68,6 +211,36 @@ class ServiceIssueImportSummaryEndpoint(BaseAPIView):
status=status.HTTP_400_BAD_REQUEST,
)
def get(self, request, slug):
try:
imports = Importer.objects.filter(workspace__slug=slug)
serializer = ImporterSerializer(imports, many=True)
return Response(serializer.data)
except Exception as e:
capture_exception(e)
return Response(
{"error": "Something went wrong please try again later"},
status=status.HTTP_400_BAD_REQUEST,
)
class UpdateServiceImportStatusEndpoint(BaseAPIView):
def post(self, request, slug, project_id, service, importer_id):
try:
importer = Importer.objects.get(
pk=importer_id,
workspace__slug=slug,
project_id=project_id,
service=service,
)
importer.status = request.data.get("status", "processing")
importer.save()
return Response(status.HTTP_200_OK)
except Importer.DoesNotExist:
return Response(
{"error": "Importer does not exist"}, status=status.HTTP_404_NOT_FOUND
)
class BulkImportIssuesEndpoint(BaseAPIView):
def post(self, request, slug, project_id, service):
@ -115,7 +288,9 @@ class BulkImportIssuesEndpoint(BaseAPIView):
Issue(
project_id=project_id,
workspace_id=project.workspace_id,
state=default_state,
state_id=issue_data.get("state")
if issue_data.get("state", False)
else default_state.id,
name=issue_data.get("name", "Issue Created through Bulk"),
description_html=issue_data.get("description_html", "<p></p>"),
description_stripped=(
@ -130,6 +305,7 @@ class BulkImportIssuesEndpoint(BaseAPIView):
sort_order=largest_sort_order,
start_date=issue_data.get("start_date", None),
target_date=issue_data.get("target_date", None),
priority=issue_data.get("priority", None),
)
)
@ -172,7 +348,29 @@ class BulkImportIssuesEndpoint(BaseAPIView):
for label_id in labels_list
]
_ = IssueLabel.objects.bulk_create(bulk_issue_labels, batch_size=100)
_ = IssueLabel.objects.bulk_create(
bulk_issue_labels, batch_size=100, ignore_conflicts=True
)
# Attach Assignees
bulk_issue_assignees = []
for issue, issue_data in zip(issues, issues_data):
assignees_list = issue_data.get("assignees_list", [])
bulk_issue_assignees = bulk_issue_assignees + [
IssueAssignee(
issue=issue,
assignee_id=assignee_id,
project_id=project_id,
workspace_id=project.workspace_id,
created_by=request.user,
updated_by=request.user,
)
for assignee_id in assignees_list
]
_ = IssueAssignee.objects.bulk_create(
bulk_issue_assignees, batch_size=100, ignore_conflicts=True
)
# Track the issue activities
IssueActivity.objects.bulk_create(
@ -241,62 +439,75 @@ class BulkImportIssuesEndpoint(BaseAPIView):
)
class ImportServiceEndpoint(BaseAPIView):
def post(self, request, slug, service):
class BulkImportModulesEndpoint(BaseAPIView):
def post(self, request, slug, project_id, service):
try:
project_id = request.data.get("project_id", False)
modules_data = request.data.get("modules_data", [])
project = Project.objects.get(pk=project_id, workspace__slug=slug)
if not project_id:
return Response(
{"error": "Project ID is required"},
status=status.HTTP_400_BAD_REQUEST,
)
workspace = Workspace.objects.get(slug=slug)
if service == "github":
data = request.data.get("data", False)
metadata = request.data.get("metadata", False)
config = request.data.get("config", False)
if not data or not metadata or not config:
return Response(
{"error": "Data, config and metadata are required"},
status=status.HTTP_400_BAD_REQUEST,
modules = Module.objects.bulk_create(
[
Module(
name=module.get("name", uuid.uuid4().hex),
description=module.get("description", ""),
start_date=module.get("start_date", None),
target_date=module.get("target_date", None),
project_id=project_id,
workspace_id=project.workspace_id,
created_by=request.user,
updated_by=request.user,
)
api_token = APIToken.objects.filter(user=request.user).first()
if api_token is None:
api_token = APIToken.objects.create(
user=request.user,
label="Importer",
workspace=workspace,
)
importer = Importer.objects.create(
service=service,
project_id=project_id,
status="queued",
initiated_by=request.user,
data=data,
metadata=metadata,
token=api_token,
config=config,
created_by=request.user,
updated_by=request.user,
)
service_importer.delay(service, importer.id)
serializer = ImporterSerializer(importer)
return Response(serializer.data, status=status.HTTP_200_OK)
return Response(
{"error": "Servivce not supported yet"},
status=status.HTTP_400_BAD_REQUEST,
for module in modules_data
],
batch_size=100,
ignore_conflicts=True,
)
except (Workspace.DoesNotExist, WorkspaceIntegration.DoesNotExist) as e:
_ = ModuleLink.objects.bulk_create(
[
ModuleLink(
module=module,
url=module_data.get("link", {}).get("url", "https://plane.so"),
title=module_data.get("link", {}).get(
"title", "Original Issue"
),
project_id=project_id,
workspace_id=project.workspace_id,
created_by=request.user,
updated_by=request.user,
)
for module, module_data in zip(modules, modules_data)
],
batch_size=100,
ignore_conflicts=True,
)
bulk_module_issues = []
for module, module_data in zip(modules, modules_data):
module_issues_list = module_data.get("module_issues_list", [])
bulk_module_issues = bulk_module_issues + [
ModuleIssue(
issue_id=issue,
module=module,
project_id=project_id,
workspace_id=project.workspace_id,
created_by=request.user,
updated_by=request.user,
)
for issue in module_issues_list
]
_ = ModuleIssue.objects.bulk_create(
bulk_module_issues, batch_size=100, ignore_conflicts=True
)
serializer = ModuleSerializer(modules, many=True)
return Response(
{"error": "Workspace Integration does not exist"},
status=status.HTTP_404_NOT_FOUND,
{"modules": serializer.data}, status=status.HTTP_201_CREATED
)
except Project.DoesNotExist:
return Response(
{"error": "Project does not exist"}, status=status.HTTP_404_NOT_FOUND
)
except Exception as e:
capture_exception(e)
@ -304,33 +515,3 @@ class ImportServiceEndpoint(BaseAPIView):
{"error": "Something went wrong please try again later"},
status=status.HTTP_400_BAD_REQUEST,
)
def get(self, request, slug):
try:
imports = Importer.objects.filter(workspace__slug=slug)
serializer = ImporterSerializer(imports, many=True)
return Response(serializer.data)
except Exception as e:
capture_exception(e)
return Response(
{"error": "Something went wrong please try again later"},
status=status.HTTP_400_BAD_REQUEST,
)
class UpdateServiceImportStatusEndpoint(BaseAPIView):
def post(self, request, slug, project_id, service, importer_id):
try:
importer = Importer.objects.get(
pk=importer_id,
workspace__slug=slug,
project_id=project_id,
service=service,
)
importer.status = request.data.get("status", "processing")
importer.save()
return Response(status.HTTP_200_OK)
except Importer.DoesNotExist:
return Response(
{"error": "Importer does not exist"}, status=status.HTTP_404_NOT_FOUND
)

16
apiserver/plane/bgtasks/importer_task.py
View File

@ -51,17 +51,29 @@ def service_importer(service, importer_id):
if user.get("import", False) == "invite"
],
batch_size=10,
ignore_conflicts=True,
)
workspace_users = User.objects.filter(
email__in=[
user.get("email").strip().lower()
for user in users
if user.get("import", False) == "invite"
or user.get("import", False) == "map"
]
)
# Add new users to Workspace and project automatically
WorkspaceMember.objects.bulk_create(
[
WorkspaceMember(member=user, workspace_id=importer.workspace_id)
for user in new_users
for user in workspace_users
],
batch_size=100,
ignore_conflicts=True,
)
ProjectMember.objects.bulk_create(
[
ProjectMember(
@ -69,7 +81,7 @@ def service_importer(service, importer_id):
workspace_id=importer.workspace_id,
member=user,
)
for user in new_users
for user in workspace_users
],
batch_size=100,
ignore_conflicts=True,

8
apiserver/plane/db/models/importer.py
View File

@ -7,7 +7,13 @@ from . import ProjectBaseModel
class Importer(ProjectBaseModel):
service = models.CharField(max_length=50, choices=(("github", "GitHub"),))
service = models.CharField(
max_length=50,
choices=(
("github", "GitHub"),
("jira", "Jira"),
),
)
status = models.CharField(
max_length=50,
choices=(

0
apiserver/plane/utils/importers/__init__.py Normal file
View File

53
apiserver/plane/utils/importers/jira.py Normal file
View File

@ -0,0 +1,53 @@
import requests
from requests.auth import HTTPBasicAuth
from sentry_sdk import capture_exception
def jira_project_issue_summary(email, api_token, project_name, hostname):
try:
auth = HTTPBasicAuth(email, api_token)
headers = {"Accept": "application/json"}
issue_url = f"https://{hostname}/rest/api/3/search?jql=project={project_name} AND issuetype=Story"
issue_response = requests.request(
"GET", issue_url, headers=headers, auth=auth
).json()["total"]
module_url = f"https://{hostname}/rest/api/3/search?jql=project={project_name} AND issuetype=Epic"
module_response = requests.request(
"GET", module_url, headers=headers, auth=auth
).json()["total"]
status_url = f"https://{hostname}/rest/api/3/status/?jql=project={project_name}"
status_response = requests.request(
"GET", status_url, headers=headers, auth=auth
).json()
labels_url = f"https://{hostname}/rest/api/3/label/?jql=project={project_name}"
labels_response = requests.request(
"GET", labels_url, headers=headers, auth=auth
).json()["total"]
users_url = (
f"https://{hostname}/rest/api/3/users/search?jql=project={project_name}"
)
users_response = requests.request(
"GET", users_url, headers=headers, auth=auth
).json()
return {
"issues": issue_response,
"modules": module_response,
"labels": labels_response,
"states": len(status_response),
"users": (
[
user
for user in users_response
if user.get("accountType") == "atlassian"
]
),
}
except Exception as e:
capture_exception(e)
return {"error": "Something went wrong could not fetch information from jira"}