From aa102acf8df5a61de15f7e453b07cfbbe81fb2d6 Mon Sep 17 00:00:00 2001 From: Struchkov Mark Date: Fri, 16 Dec 2022 21:49:40 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BF=D0=B8=D1=81?= =?UTF-8?q?=D0=B0=D0=BB=20=D0=BB=D0=BE=D0=B3=D0=B8=D0=BA=D1=83=20=D0=BE?= =?UTF-8?q?=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B8=20=D0=BD=D0=BE?= =?UTF-8?q?=D0=B2=D1=8B=D1=85=20=D0=9F=D0=A0.=20=D0=A3=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D1=80=D0=B8=D0=BB=20=D0=B8=D1=81=D0=BF=D0=BE=D0=BB=D1=8C=D0=B7?= =?UTF-8?q?=D1=83=D1=8F=20=D1=84=D0=BE=D1=80=D0=BA-=D0=B4=D0=B6=D0=BE?= =?UTF-8?q?=D0=B9=D0=BD-=D0=BF=D1=83=D0=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/parser/MergeRequestParser.java | 82 +++++++++++++------ .../forktask/GetMergeRequestFromGitlab.java | 49 +++++++++++ .../gitlab/scheduler/SchedulerService.java | 12 +-- gitlab-app/src/main/resources/application.yml | 7 +- 4 files changed, 115 insertions(+), 35 deletions(-) create mode 100644 bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java index 221f89e..f1088b4 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java @@ -9,6 +9,7 @@ import dev.struchkov.bot.gitlab.context.service.MergeRequestsService; import dev.struchkov.bot.gitlab.context.service.ProjectService; import dev.struchkov.bot.gitlab.core.config.properties.GitlabProperty; import dev.struchkov.bot.gitlab.core.config.properties.PersonProperty; +import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetMergeRequestFromGitlab; import dev.struchkov.bot.gitlab.core.utils.StringUtils; import dev.struchkov.bot.gitlab.sdk.domain.CommitJson; import dev.struchkov.bot.gitlab.sdk.domain.MergeRequestJson; @@ -18,12 +19,19 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.core.convert.ConversionService; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -36,7 +44,6 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; @RequiredArgsConstructor public class MergeRequestParser { - public static final Integer COUNT = 100; private static final Set OLD_STATUSES = Set.of( MergeRequestState.MERGED, MergeRequestState.OPENED, MergeRequestState.CLOSED ); @@ -47,6 +54,18 @@ public class MergeRequestParser { private final ConversionService conversionService; private final PersonProperty personProperty; + private ForkJoinPool forkJoinPool; + + @PreDestroy + public void preDestroy() { + forkJoinPool.shutdown(); + } + + @PostConstruct + public void postConstruct() { + forkJoinPool = new ForkJoinPool(4); + } + public void parsingOldMergeRequest() { log.debug("Старт обработки старых MR"); final Set existIds = mergeRequestsService.getAllId(OLD_STATUSES); @@ -73,19 +92,9 @@ public class MergeRequestParser { log.debug("Старт обработки новых MR"); final Set projectIds = projectService.getAllIds(); - for (Long projectId : projectIds) { - projectProcessing(projectId); - } - - log.debug("Конец обработки новых MR"); - } - - private void projectProcessing(Long projectId) { - int page = 1; - List mergeRequestJsons = getMergeRequestJsons(projectId, page); - - while (checkNotEmpty(mergeRequestJsons)) { + final List mergeRequestJsons = getMergeRequests(projectIds); + if (checkNotEmpty(mergeRequestJsons)) { final Set jsonIds = mergeRequestJsons.stream() .map(MergeRequestJson::getId) .collect(Collectors.toSet()); @@ -107,9 +116,43 @@ public class MergeRequestParser { log.trace("Пачка новых MR обработана и отправлена на сохранение. Количество: {} шт.", newMergeRequests.size()); mergeRequestsService.createAll(newMergeRequests); } - - mergeRequestJsons = getMergeRequestJsons(projectId, page++); } + + log.debug("Конец обработки новых MR"); + } + + /** + * Позволяет получить MR для переданных идентификаторов проектов. + * + * @param projectIds идентификаторы проектов + * @return полученные у GitLab MergeRequests + */ + private List getMergeRequests(Set projectIds) { + final List>> tasks = projectIds.stream() + .map(projectId -> new GetMergeRequestFromGitlab(projectId, gitlabProperty.getUrlPullRequestOpen(), personProperty.getToken())) + .map(forkJoinPool::submit) + .collect(Collectors.toList()); + + final List mergeRequestJsons = new ArrayList<>(); + Iterator>> iterator = tasks.iterator(); + while (!tasks.isEmpty()) { + while (iterator.hasNext()) { + final ForkJoinTask> task = iterator.next(); + if (task.isDone()) { + final List jsons; + try { + jsons = task.get(); + mergeRequestJsons.addAll(jsons); + } catch (InterruptedException | ExecutionException e) { + log.error(e.getMessage(), e); + Thread.currentThread().interrupt(); + } + iterator.remove(); + } + } + iterator = tasks.iterator(); + } + return mergeRequestJsons; } private static void personMapping(List newMergeRequests) { @@ -150,15 +193,6 @@ public class MergeRequestParser { } } - private List getMergeRequestJsons(Long projectId, int page) { - final List jsons = HttpParse.request(MessageFormat.format(gitlabProperty.getUrlPullRequestOpen(), projectId, page)) - .header(StringUtils.H_PRIVATE_TOKEN, personProperty.getToken()) - .header(ACCEPT) - .executeList(MergeRequestJson.class); - log.trace("Получено {} шт потенциально новых MR для проекта id:'{}' ", jsons.size(), projectId); - return jsons; - } - private Optional getMergeRequest(IdAndStatusPr existId) { final String mrUrl = MessageFormat.format(gitlabProperty.getUrlPullRequest(), existId.getProjectId(), existId.getTwoId()); return HttpParse.request(mrUrl) diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java new file mode 100644 index 0000000..7c460a5 --- /dev/null +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java @@ -0,0 +1,49 @@ +package dev.struchkov.bot.gitlab.core.service.parser.forktask; + +import dev.struchkov.bot.gitlab.core.utils.StringUtils; +import dev.struchkov.bot.gitlab.sdk.domain.MergeRequestJson; +import dev.struchkov.haiti.utils.network.HttpParse; +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.List; +import java.util.concurrent.RecursiveTask; + +import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; + +@Slf4j +@AllArgsConstructor +@RequiredArgsConstructor +public class GetMergeRequestFromGitlab extends RecursiveTask> { + + private static final int PAGE_COUNT = 100; + + private final long projectId; + private int pageNumber = 0; + private final String urlMrOpen; + private final String personToken; + + @Override + protected List compute() { + final List mergeRequestJsons = getMergeRequestJsons(urlMrOpen, projectId, pageNumber, personToken); + if (mergeRequestJsons.size() == PAGE_COUNT) { + final GetMergeRequestFromGitlab newTask = new GetMergeRequestFromGitlab(projectId, pageNumber + 1, urlMrOpen, personToken); + newTask.fork(); + final List result = newTask.join(); + mergeRequestJsons.addAll(result); + } + return mergeRequestJsons; + } + + private List getMergeRequestJsons(String url, Long projectId, int page, String personToken) { + final List jsons = HttpParse.request(MessageFormat.format(url, projectId, page, PAGE_COUNT)) + .header(StringUtils.H_PRIVATE_TOKEN, personToken) + .header(ACCEPT) + .executeList(MergeRequestJson.class); + log.trace("Получено {} шт потенциально новых MR для проекта id:'{}' ", jsons.size(), projectId); + return jsons; + } + +} diff --git a/gitlab-app/src/main/java/dev/struchkov/bot/gitlab/scheduler/SchedulerService.java b/gitlab-app/src/main/java/dev/struchkov/bot/gitlab/scheduler/SchedulerService.java index 7d01fe0..c69fb81 100644 --- a/gitlab-app/src/main/java/dev/struchkov/bot/gitlab/scheduler/SchedulerService.java +++ b/gitlab-app/src/main/java/dev/struchkov/bot/gitlab/scheduler/SchedulerService.java @@ -27,12 +27,12 @@ public class SchedulerService { log.debug("Запуск процесса обновления данных"); mergeRequestParser.parsingOldMergeRequest(); mergeRequestParser.parsingNewMergeRequest(); -// pipelineParser.scanOldPipeline(); -// pipelineParser.scanNewPipeline(); -// discussionParser.scanOldDiscussions(); -// discussionParser.scanNewDiscussion(); -// cleanService.cleanOldPipelines(); -// cleanService.cleanOldMergedRequests(); + pipelineParser.scanOldPipeline(); + pipelineParser.scanNewPipeline(); + discussionParser.scanOldDiscussions(); + discussionParser.scanNewDiscussion(); + cleanService.cleanOldPipelines(); + cleanService.cleanOldMergedRequests(); log.debug("Конец процесса обновления данных"); } diff --git a/gitlab-app/src/main/resources/application.yml b/gitlab-app/src/main/resources/application.yml index 2142301..8aeb28f 100644 --- a/gitlab-app/src/main/resources/application.yml +++ b/gitlab-app/src/main/resources/application.yml @@ -37,7 +37,7 @@ gitlab-bot: base-url: ${GITLAB_URL} replaceUrl: ${GITLAB_REPLACE_URL} url-project: ${GITLAB_URL}/api/v4/projects?page={0, number, integer}&per_page=100 - url-pull-request-open: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests?state=opened&page={1, number, integer}&per_page=100" + url-pull-request-open: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests?state=opened&page={1, number, integer}&per_page={2, number, integer}" url-pull-request-close: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests?state=closed&page={1, number, integer}&per_page=100" url-pull-request-comment: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/notes?&page={2,number,#}&per_page=100" url-pull-request: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}" @@ -51,7 +51,4 @@ gitlab-bot: url-commit: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/commits?&page=1&per_page=1" url-new-note: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}/notes?body={3}" url-discussion: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions?&page={2,number,#}&per_page=100" - url-one-discussion: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}" -logging: - level: - "dev.struchkov": trace \ No newline at end of file + url-one-discussion: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}" \ No newline at end of file