From 422de7385469979f08faf2d3a9804cfba3b9a35b Mon Sep 17 00:00:00 2001 From: Struchkov Mark Date: Tue, 20 Dec 2022 19:55:17 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20?= =?UTF-8?q?=D0=B8=D1=81=D0=BF=D0=BE=D0=BB=D1=8C=D0=B7=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20ForkJoinPool=20=D0=B4=D0=BB=D1=8F=20=D0=BE?= =?UTF-8?q?=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D1=80=D1=8B=D1=85=20MR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/parser/MergeRequestParser.java | 42 +++++++++++-------- ... => GetAllMergeRequestForProjectTask.java} | 4 +- .../forktask/GetSingleMergeRequestTask.java | 37 ++++++++++++++++ 3 files changed, 64 insertions(+), 19 deletions(-) rename bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/{GetMergeRequestTask.java => GetAllMergeRequestForProjectTask.java} (87%) create mode 100644 bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetSingleMergeRequestTask.java diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java index 635b374..d4874aa 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java @@ -9,7 +9,8 @@ import dev.struchkov.bot.gitlab.context.service.MergeRequestsService; import dev.struchkov.bot.gitlab.context.service.ProjectService; import dev.struchkov.bot.gitlab.core.config.properties.GitlabProperty; import dev.struchkov.bot.gitlab.core.config.properties.PersonProperty; -import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetMergeRequestTask; +import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetAllMergeRequestForProjectTask; +import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetSingleMergeRequestTask; import dev.struchkov.bot.gitlab.core.utils.StringUtils; import dev.struchkov.bot.gitlab.sdk.domain.CommitJson; import dev.struchkov.bot.gitlab.sdk.domain.MergeRequestJson; @@ -32,6 +33,7 @@ import java.util.stream.Stream; import static dev.struchkov.haiti.utils.Checker.checkNotEmpty; import static dev.struchkov.haiti.utils.Checker.checkNotNull; +import static dev.struchkov.haiti.utils.concurrent.ForkJoinUtils.pullTaskResult; import static dev.struchkov.haiti.utils.concurrent.ForkJoinUtils.pullTaskResults; import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; @@ -71,10 +73,7 @@ public class MergeRequestParser { log.debug("Старт обработки старых MR"); final Set existIds = mergeRequestsService.getAllId(OLD_STATUSES); - final List mergeRequests = existIds.stream() - .map(this::getMergeRequest) - .filter(Optional::isPresent) - .map(Optional::get) + final List newMergeRequests = getOldMergeRequests(existIds).stream() .map(mergeRequestJson -> { final MergeRequest newMergeRequest = conversionService.convert(mergeRequestJson, MergeRequest.class); parsingCommits(newMergeRequest); @@ -82,13 +81,30 @@ public class MergeRequestParser { }) .collect(Collectors.toList()); - if (checkNotEmpty(mergeRequests)) { - personMapping(mergeRequests); - mergeRequestsService.updateAll(mergeRequests); + if (checkNotEmpty(newMergeRequests)) { + personMapping(newMergeRequests); + mergeRequestsService.updateAll(newMergeRequests); } log.debug("Конец обработки старых MR"); } + private List getOldMergeRequests(Set existIds) { + final List>> tasks = existIds.stream() + .map( + existId -> new GetSingleMergeRequestTask( + gitlabProperty.getUrlMergeRequest(), + existId.getProjectId(), + existId.getTwoId(), + personProperty.getToken() + ) + ).map(forkJoinPool::submit) + .collect(Collectors.toList()); + + return pullTaskResult(tasks).stream() + .flatMap(Optional::stream) + .collect(Collectors.toList()); + } + public void parsingNewMergeRequest() { log.debug("Старт обработки новых MR"); final Set projectIds = projectService.getAllIds(); @@ -130,7 +146,7 @@ public class MergeRequestParser { */ private List getMergeRequests(Set projectIds) { final List>> tasks = projectIds.stream() - .map(projectId -> new GetMergeRequestTask(projectId, gitlabProperty.getUrlMergeRequestOpen(), personProperty.getToken())) + .map(projectId -> new GetAllMergeRequestForProjectTask(projectId, gitlabProperty.getUrlMergeRequestOpen(), personProperty.getToken())) .map(forkJoinPool::submit) .collect(Collectors.toList()); @@ -175,12 +191,4 @@ public class MergeRequestParser { } } - private Optional getMergeRequest(IdAndStatusPr existId) { - final String mrUrl = MessageFormat.format(gitlabProperty.getUrlMergeRequest(), existId.getProjectId(), existId.getTwoId()); - return HttpParse.request(mrUrl) - .header(ACCEPT) - .header(StringUtils.H_PRIVATE_TOKEN, personProperty.getToken()) - .execute(MergeRequestJson.class); - } - } diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestTask.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllMergeRequestForProjectTask.java similarity index 87% rename from bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestTask.java rename to bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllMergeRequestForProjectTask.java index 0d8b474..4013a45 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestTask.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllMergeRequestForProjectTask.java @@ -17,7 +17,7 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; @Slf4j @AllArgsConstructor @RequiredArgsConstructor -public class GetMergeRequestTask extends RecursiveTask> { +public class GetAllMergeRequestForProjectTask extends RecursiveTask> { private static final int PAGE_COUNT = 100; @@ -32,7 +32,7 @@ public class GetMergeRequestTask extends RecursiveTask> { Thread.sleep(100); final List mergeRequestJsons = getMergeRequestJsons(); if (mergeRequestJsons.size() == PAGE_COUNT) { - final GetMergeRequestTask newTask = new GetMergeRequestTask(projectId, pageNumber + 1, urlMrOpen, gitlabToken); + final GetAllMergeRequestForProjectTask newTask = new GetAllMergeRequestForProjectTask(projectId, pageNumber + 1, urlMrOpen, gitlabToken); newTask.fork(); mergeRequestJsons.addAll(newTask.join()); } diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetSingleMergeRequestTask.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetSingleMergeRequestTask.java new file mode 100644 index 0000000..45bf499 --- /dev/null +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetSingleMergeRequestTask.java @@ -0,0 +1,37 @@ +package dev.struchkov.bot.gitlab.core.service.parser.forktask; + +import dev.struchkov.bot.gitlab.core.utils.StringUtils; +import dev.struchkov.bot.gitlab.sdk.domain.MergeRequestJson; +import dev.struchkov.haiti.utils.network.HttpParse; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.Optional; +import java.util.concurrent.RecursiveTask; + +import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; + +@Slf4j +@RequiredArgsConstructor +public class GetSingleMergeRequestTask extends RecursiveTask> { + + private final String urlMr; + private final long projectId; + private final long mrTwoId; + private final String gitlabToken; + + @Override + @SneakyThrows + protected Optional compute() { + Thread.sleep(100); + final String mrUrl = MessageFormat.format(urlMr, projectId, mrTwoId); + return HttpParse.request(mrUrl) + .header(ACCEPT) + .header(StringUtils.H_PRIVATE_TOKEN, gitlabToken) + .execute(MergeRequestJson.class); + } + + +}