Initial jira migration tool.

- models
- db structures
- fetch data job
- import data job
This commit is contained in:
ba1ash
2025-12-08 10:55:50 +01:00
parent d8ee971298
commit ec36f338f1
22 changed files with 722 additions and 4 deletions
+1 -2
View File
@@ -36,8 +36,7 @@ class StatusesController < ApplicationController
before_action :require_admin
def index
@statuses = Status.page(page_param)
.per_page(per_page_param)
@statuses = Status.page(page_param).per_page(per_page_param)
render action: "index", layout: false if request.xhr?
end
+3
View File
@@ -0,0 +1,3 @@
class Jira < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraField < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraImport < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraIssue < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraIssueType < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraPriority < ApplicationRecord
end
+10
View File
@@ -0,0 +1,10 @@
class JiraProject < ApplicationRecord
def to_op_attributes
{
name: payload["name"],
identifier: payload["key"].downcase,
parent_id: "",
workspace_type: "project"
}
end
end
+3
View File
@@ -0,0 +1,3 @@
class JiraProjectType < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraStatus < ApplicationRecord
end
+3
View File
@@ -0,0 +1,3 @@
class JiraStatusCategory < ApplicationRecord
end
+18
View File
@@ -0,0 +1,18 @@
class JiraUser < ApplicationRecord
def self.groups
all.map { |x| x.payload["groups"]["items"] }.flatten.uniq {|x| x["name"]}
end
def to_op_attributes
firstname = payload["displayName"].split(" ")[0..-2].join(" ")
lastname = payload["displayName"].split(" ")[-1]
{
login: payload["name"],
password: SecureRandom.uuid,
firstname:,
lastname:,
mail: payload["emailAddress"],
status: payload["active"] ? :active : :locked
}
end
end
@@ -0,0 +1,5 @@
class OpenProjectJiraReference < ApplicationRecord
def model
op_entity_table.constantize.find(op_entity_id)
end
end
+133
View File
@@ -0,0 +1,133 @@
class J
=begin
curl --request GET \
--url 'https://jira-software.local/rest/api/2/mypermissions' \
--header 'Accept: application/json' \
--header 'Authorization: Bearer <personal_access_token>'
curl --request GET \
--url 'https://jira-software.local/rest/api/2/search?jql=issuekey=PROCESS1-3' \
--user 'pavel.balashou:pavel.balashou' \
--header 'Accept: application/json' \
--header 'Authorization: Bearer <personal_access_token>'
=end
def initialize(
url:,
personal_access_token:
)
@httpx = OpenProject
.httpx
.plugin(:basic_auth)
.with(headers: { "accept" => "application/json" })
.bearer_auth(personal_access_token)
@url = url
end
# response["permissions"]["SYSTEM_ADMIN"]["havePermission"] == true
def mypermissions
@httpx.get("#{@url}/rest/api/2/mypermissions").json
end
def index_condition_summary
@httpx.get("#{@url}/rest/api/2/index/summary").json
end
def server_info
@httpx.get("#{@url}/rest/api/2/serverInfo").json
end
def all_cluster_nodes
@httpx.get("#{@url}/rest/api/2/cluster/nodes").json
end
def issues(jql: nil,
start_at: 0,
max_results: 100,
fields: "*all",
expand: "changelog")
@httpx.get(
"#{@url}/rest/api/2/search",
params: {
jql:,
startAt: start_at,
maxResults: max_results,
fields:,
expand:
}
).json
end
def projects(expand = "description,projectKeys")
@httpx.get("#{@url}/rest/api/2/project", params: { "expand" => expand }).json
end
def project_types
@httpx.get("#{@url}/rest/api/2/project/type").json
end
def issue_types
@httpx.get("#{@url}/rest/api/2/issuetype").json
end
def issue_types_schemes
@httpx.get("#{@url}/rest/api/2/issuetypescheme").json
end
def workflows
@httpx.get("#{@url}/rest/api/2/workflow").json
end
def workflowschemes
@httpx.get("#{@url}/rest/api/2/workflowscheme").json
end
def statuses
@httpx.get("#{@url}/rest/api/2/status").json
end
def status_categories
@httpx.get("#{@url}/rest/api/2/statuscategory").json
end
def permissions
@httpx.get("#{@url}/rest/api/2/permissions").json
end
def permission_schemes
@httpx.get("#{@url}/rest/api/2/permissionschemes").json
end
def priorities
@httpx.get("#{@url}/rest/api/2/priority").json
end
def permission_schemes
@httpx.get("#{@url}/rest/api/2/priorityschemes").json
end
def roles
@httpx.get("#{@url}/rest/api/2/role").json
end
def fields
@httpx.get("#{@url}/rest/api/2/field").json
end
def users_search(username: ".", start_at: 0, max_results: 50)
@httpx.get("#{@url}/rest/api/2/user/search", params: {
"username" => username,
startAt: start_at,
maxResults: max_results,
includeActive: true,
includeInactive: true
}).json
end
def user_by_key(key:)
@httpx.get("#{@url}/rest/api/2/user", params: { key:, expand: "groups" }).json
end
def groups(query: ".", start_at: 0, max_results: 50)
@httpx.get("#{@url}/rest/api/2/groups/picker", params: { query:, startAt: start_at, maxResults: max_results }).json
end
end
+32
View File
@@ -0,0 +1,32 @@
# frozen_string_literal: true
#-- copyright
# OpenProject is an open source project management software.
# Copyright (C) the OpenProject GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License version 3.
#
# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
# Copyright (C) 2006-2013 Jean-Philippe Lang
# Copyright (C) 2010-2013 the ChiliProject Team
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# See COPYRIGHT and LICENSE files for more details.
#++
class Statuses::DeleteService < BaseServices::Delete
end
+105
View File
@@ -0,0 +1,105 @@
class JiraImportJob < ApplicationJob
include GoodJob::ActiveJobExtensions::Concurrency
good_job_control_concurrency_with(
total_limit: 2,
enqueue_limit: 1,
perform_limit: 1,
key: -> { "JiraImportJob-#{arguments.last}" }
)
def perform(jira_id)
ActiveRecord::Base.transaction do
jira = Jira.find(jira_id)
# IMPORT USERS GROUPS MEMBERSHIPS
jira_users = JiraUser.where(jira_id: jira.id)
# group_name => member_ids
groups = {}
jira_users.each do |jira_user|
call = Users::CreateService
.new(user: User.system)
.call(jira_user.to_op_attributes)
ref = nil
call.on_success do |result|
user_id = call.result.id
ref = OpenProjectJiraReference.create!(
op_entity_id: user_id,
op_entity_table: "User",
jira_id: jira.id,
jira_entity_id: jira_user.id,
jira_entity_table: "JiraUser",
created: true
)
jira_user
.payload["groups"]["items"]
.each do |item|
group = item["name"]
groups[group] = Set.new unless groups.key?(group)
groups[group] << user_id
end
end
call.on_failure do |result|
binding.pry
end
end
groups.each do |name, member_ids|
call = Groups::CreateService
.new(user: User.system)
.call(name:)
call.on_success do |result|
group = result.result
ref = OpenProjectJiraReference.create!(
op_entity_id: group.id,
op_entity_table: "Group",
jira_id: jira.id,
jira_entity_id: nil,
jira_entity_table: nil,
created: true
)
if member_ids.present?
add_users_call = Groups::AddUsersService
.new(group, current_user: User.system)
.call(ids: member_ids, send_notifications: false)
end
end
call.on_failure do |result|
binding.pry
end
end
# IMPORT STATUSES
binding.pry
JiraStatus.all.each do |jira_status|
status = Status.create!(name: "J-#{jira_status.payload['name']}")
ref = OpenProjectJiraReference.create!(
op_entity_id: status.id,
op_entity_table: "Status",
jira_id: jira.id,
jira_entity_id: jira_status.id,
jira_entity_table: "JiraStatus",
)
end
# create status
# create reference
# cleanup
binding.pry
raise ActiveRecord::Rollback
# OpenProjectJiraReference.all.map(&:model).each do |model|
# delete_service = "#{model.class.to_s.pluralize}::DeleteService".constantize
# delete_service
# .new(user: User.system, model:)
# .call
# end
# OpenProjectJiraReference.destroy_all
end
end
end
+208
View File
@@ -0,0 +1,208 @@
class JiraSyncJob < ApplicationJob
include GoodJob::ActiveJobExtensions::Concurrency
good_job_control_concurrency_with(
total_limit: 2,
enqueue_limit: 1,
perform_limit: 1,
key: -> { "JiraSyncJob-#{arguments.last}" }
)
=begin
jira= Jira.new
jira.url = "https://jira-software.local/"
jira.personal_access_token = "<personal_access_token>"
jira.save
j = J.new(url: jira.url, personal_access_token: jira.personal_access_token)
JiraSyncJob.new.perform(1)
=end
def perform(jira_id)
ActiveRecord::Base.transaction do
jira = Jira.find(jira_id)
jira_import = JiraImport.find_or_create_by!(status: "init_sync_in_progress", jira_id: jira_id)
jira_import.import_time_point ||= Time.now
jira_import.save
jira_import_id = jira_import.id
updated_at = Time.now
created_at = updated_at
# PROJECTS SYNC
j = J.new(url: jira.url, personal_access_token: jira.personal_access_token)
projects_upsert_data = j.projects.map do |p|
{
payload: p,
jira_id:,
jira_project_id: p.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraProject.upsert_all(projects_upsert_data, unique_by: [:jira_id, :jira_project_id])
# PROJECT ISSUES SYNC
JiraProject.where(jira_id:).each do |jira_project|
already_synced_issue_ids = JiraIssue.where(jira_import_id:, jira_project_id: jira_project.id).pluck(Arel.sql("payload->>'id'"))
jql = "project=#{jira_project.payload["key"]} AND updated <= '#{jira_import.import_time_point.strftime("%Y-%m-%d %H:%M")}'"
# TODO Use POST not GET to avoid: having a long list of issues can exceed a server limit for request URI length.
jql << " AND id NOT IN (#{already_synced_issue_ids.join(",")})" if already_synced_issue_ids.any?
result = j.issues(jql: ,
start_at: 0,
max_results: 5)
total = result["total"]
start_at = result["startAt"]
max_results = result["maxResults"]
issues = result["issues"]
issues_upsert_data = result["issues"].map do |issue|
{
payload: issue,
jira_id: jira_id,
jira_project_id: jira_project.id,
jira_issue_id: issue.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraIssue.upsert_all(issues_upsert_data, unique_by: [:jira_id, :jira_project_id, :jira_issue_id])
while(total > start_at + max_results)
start_at = start_at + max_results
result = j.issues(jql:,
start_at:,
max_results: 5)
total = result["total"]
start_at = result["startAt"]
max_results = result["maxResults"]
issues = result["issues"]
issues_upsert_data = result["issues"].map do |issue|
{
payload: issue,
jira_id: jira_id,
jira_project_id: jira_project.id,
jira_issue_id: issue.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraIssue.upsert_all(issues_upsert_data, unique_by: [:jira_id, :jira_project_id, :jira_issue_id])
end
end
# USERS with GROUP memberships SYNC
start_at = 0
max_results = 1 # It should be 1000 to reduce the number of requests
jira_users = j.users_search(start_at: , max_results: )
users_upsert_data = jira_users.map do |jira_user_from_search|
jira_user_key = jira_user_from_search.fetch('key')
# here we send a direct user request to get group memberships
# which are not returned by users_search endpoint
jira_user_by_key = j.user_by_key(key: jira_user_key)
{
payload: jira_user_by_key,
jira_id: jira_id,
jira_import_id: jira_import.id,
jira_user_key: ,
created_at:,
updated_at:
}
end
upsert_result = JiraUser.upsert_all(users_upsert_data, unique_by: [:jira_id, :jira_user_key])
while(jira_users.any?)
start_at = start_at + jira_users.count
jira_users = j.users_search(start_at: , max_results: )
users_upsert_data = jira_users.map do |jira_user_from_search|
jira_user_key = jira_user_from_search.fetch('key')
# here we send a direct user request to get group memberships
# which are not returned by users_search endpoint
jira_user_by_key = j.user_by_key(key: jira_user_key)
{
payload: jira_user_by_key,
jira_id: jira_id,
jira_import_id: jira_import.id,
jira_user_key:,
created_at:,
updated_at:
}
end
upsert_result = JiraUser.upsert_all(users_upsert_data, unique_by: [:jira_id, :jira_user_key])
end
# ISSUE TYPES SYNC
issue_types = j.issue_types
issue_types_upsert_data = issue_types.map do |issue_type|
{
payload: issue_type,
jira_id: jira_id,
jira_issue_type_id: issue_type.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraIssueType.upsert_all(issue_types_upsert_data, unique_by: [:jira_id, :jira_issue_type_id])
# PRIORITIES SYNC
priorities = j.priorities
priorities_upsert_data = priorities.map do |priority|
{
payload: priority,
jira_id: jira_id,
jira_priority_id: priority.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraPriority.upsert_all(priorities_upsert_data, unique_by: [:jira_id, :jira_priority_id])
# STATUSES SYNC
statuses = j.statuses
statuses_upsert_data = statuses.map do |status|
{
payload: status,
jira_id: jira_id,
jira_status_id: status.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraStatus.upsert_all(statuses_upsert_data, unique_by: [:jira_id, :jira_status_id])
# STATUS CATEGORIES SYNC
status_categories = j.status_categories
status_categories_upsert_data = status_categories.map do |status_category|
{
payload: status_category,
jira_id: jira_id,
jira_status_category_id: status_category.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraStatusCategory.upsert_all(status_categories_upsert_data, unique_by: [:jira_id, :jira_status_category_id])
# FIELDS SYNC
fields = j.fields
fields_upsert_data = fields.map do |field|
{
payload: field,
jira_id: jira_id,
jira_field_id: field.fetch("id"),
jira_import_id: jira_import.id,
created_at:,
updated_at:
}
end
upsert_result = JiraField.upsert_all(fields_upsert_data, unique_by: [:jira_id, :jira_field_id])
jira_import.status = "init_sync_done"
jira_import.save!
end
end
end
@@ -0,0 +1,119 @@
# frozen_string_literal: true
class CreateJiraMigrationTables < ActiveRecord::Migration[8.0]
def change
create_table :jiras do |t|
t.string :url
t.string :personal_access_token
t.timestamps
end
create_table :jira_imports do |t|
t.string :status
t.timestamp :import_time_point
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
end
create_table :jira_projects do |t|
t.jsonb :payload
t.string :jira_project_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_project_id], unique: true
t.timestamps
end
create_table :jira_project_types do |t|
t.jsonb :payload
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.timestamps
end
create_table :jira_issues do |t|
t.jsonb :payload
t.string :jira_project_id
t.string :jira_issue_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_project_id, :jira_issue_id], unique: true
t.timestamps
end
create_table :jira_issue_types do |t|
t.jsonb :payload
t.string :jira_issue_type_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_issue_type_id], unique: true
t.timestamps
end
create_table :jira_priorities do |t|
t.jsonb :payload
t.string :jira_priority_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_priority_id], unique: true
t.timestamps
end
create_table :jira_statuses do |t|
t.jsonb :payload
t.string :jira_status_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_status_id], unique: true
t.timestamps
end
create_table :jira_status_categories do |t|
t.jsonb :payload
t.string :jira_status_category_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_status_category_id], unique: true
t.timestamps
end
create_table :jira_fields do |t|
t.jsonb :payload
t.string :jira_field_id
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_field_id], unique: true
t.timestamps
end
create_table :jira_users do |t|
t.jsonb :payload
t.string :jira_user_key
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:jira_id, :jira_user_key], unique: true
t.timestamps
end
create_table :open_project_jira_references do |t|
t.string :op_entity_id
t.string :op_entity_table
t.string :jira_entity_id
t.string :jira_entity_table
t.boolean :new_op_record
t.references :jira, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.references :jira_import, foreign_key: { on_delete: :cascade, on_update: :cascade }
t.index [:op_entity_id, :op_entity_table], unique: true
t.timestamps
end
end
end
+2
View File
@@ -103,6 +103,8 @@ services:
POSTGRES_DB: ${DB_DATABASE:-openproject}
networks:
- network
ports:
- "4444:5432"
cache:
image: memcached
@@ -0,0 +1,35 @@
services:
db-jira:
image: postgres:17
restart: unless-stopped
volumes:
- "pgdata:/var/lib/postgresql/data"
environment:
- POSTGRES_DB=jira-software
- POSTGRES_USER=jira-software
- POSTGRES_PASSWORD=jira-software
networks:
- gateway
jira-software:
image: atlassian/jira-software:10.3.11
labels:
- "traefik.enable=true"
- "traefik.http.routers.jira-software.rule=Host(`jira-software.local`)"
- "traefik.http.routers.jira-software.service=jira-software-service"
- "traefik.http.routers.jira-software.tls=true"
- "traefik.http.services.jira-software-service.loadbalancer.server.port=8080"
- "traefik.http.routers.jira-software.tls.certresolver=step"
networks:
- gateway
volumes:
- jiraVolume:/var/atlassian/application-data/jira
depends_on:
- db-jira
networks:
gateway:
external: true
name: gateway
volumes:
jiraVolume:
pgdata:
+26
View File
@@ -0,0 +1,26 @@
services:
youtrack:
image: jetbrains/youtrack:2025.2.98373
labels:
- "traefik.enable=true"
- "traefik.http.routers.youtrack.rule=Host(`youtrack.local`)"
- "traefik.http.routers.youtrack.service=youtrack-service"
- "traefik.http.routers.youtrack.tls=true"
- "traefik.http.services.youtrack-service.loadbalancer.server.port=8080"
- "traefik.http.routers.youtrack.tls.certresolver=step"
networks:
- gateway
volumes:
- youtrack-data:/opt/youtrack/data
- youtrack-conf:/opt/youtrack/conf
- youtrack-logs:/opt/youtrack/logs
- youtrack-backups:/opt/youtrack/backups
networks:
gateway:
external: true
name: gateway
volumes:
youtrack-data:
youtrack-conf:
youtrack-logs:
youtrack-backups:
@@ -53,8 +53,7 @@ module JobStatus
##
# Get the current status object, if any
def job_status
::JobStatus::Status
.find_by(job_id:)
::JobStatus::Status.find_by(job_id:)
end
##