From 77d9e6aba400c4f394f8a212410965f3b2d53fb0 Mon Sep 17 00:00:00 2001 From: Struchkov Mark Date: Fri, 16 Dec 2022 23:26:45 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A3=D1=81=D0=BA=D0=BE=D1=80=D0=B8=D0=BB=20?= =?UTF-8?q?=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D1=83=20=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D1=8B=D1=85=20=D0=BF=D0=B0=D0=B9=D0=BF=D0=BB=D0=B0?= =?UTF-8?q?=D0=B9=D0=BD=D0=BE=D0=B2=20=D1=81=20=D0=BF=D0=BE=D0=BC=D0=BE?= =?UTF-8?q?=D1=89=D1=8C=D1=8E=20ForkJoinPool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../context/service/PipelineService.java | 3 + .../bot/gitlab/core/config/CoreConfig.java | 16 +++ .../convert/PipelineJsonConverter.java | 2 + .../service/impl/PipelineServiceImpl.java | 8 ++ .../service/parser/MergeRequestParser.java | 58 ++++----- .../core/service/parser/PipelineParser.java | 112 +++++++++++------- ...omGitlab.java => GetMergeRequestTask.java} | 17 ++- .../parser/forktask/GetPipelineShortTask.java | 51 ++++++++ .../parser/forktask/GetPipelineTask.java | 37 ++++++ .../bot/gitlab/core/utils/PoolUtils.java | 60 ++++++++++ gitlab-app/src/main/resources/application.yml | 2 +- .../bot/gitlab/sdk/domain/PipelineJson.java | 4 + .../gitlab/sdk/domain/PipelineShortJson.java | 41 +++++++ 13 files changed, 317 insertions(+), 94 deletions(-) create mode 100644 bot-core/src/main/java/dev/struchkov/bot/gitlab/core/config/CoreConfig.java rename bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/{GetMergeRequestFromGitlab.java => GetMergeRequestTask.java} (65%) create mode 100644 bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineShortTask.java create mode 100644 bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineTask.java create mode 100644 bot-core/src/main/java/dev/struchkov/bot/gitlab/core/utils/PoolUtils.java create mode 100644 gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineShortJson.java diff --git a/bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/PipelineService.java b/bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/PipelineService.java index bd0a877..573b20d 100644 --- a/bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/PipelineService.java +++ b/bot-context/src/main/java/dev/struchkov/bot/gitlab/context/service/PipelineService.java @@ -8,6 +8,7 @@ import lombok.NonNull; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import java.util.List; import java.util.Set; /** @@ -19,6 +20,8 @@ public interface PipelineService { Pipeline create(@NonNull Pipeline pipeline); + List createAll(@NonNull List newPipelines); + Pipeline update(@NonNull Pipeline pipeline); Page getAllByStatuses(@NonNull Set statuses, @NonNull Pageable pagination); diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/config/CoreConfig.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/config/CoreConfig.java new file mode 100644 index 0000000..9ab69b3 --- /dev/null +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/config/CoreConfig.java @@ -0,0 +1,16 @@ +package dev.struchkov.bot.gitlab.core.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.concurrent.ForkJoinPool; + +@Configuration +public class CoreConfig { + + @Bean("parserPool") + public ForkJoinPool parserPool() { + return new ForkJoinPool(4); + } + +} diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/convert/PipelineJsonConverter.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/convert/PipelineJsonConverter.java index f636080..a8bb621 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/convert/PipelineJsonConverter.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/convert/PipelineJsonConverter.java @@ -39,6 +39,8 @@ public class PipelineJsonConverter implements Converter pipeline.setWebUrl(source.getWebUrl()); pipeline.setStatus(convertStatus(source.getStatus())); pipeline.setPerson(convertPerson.convert(source.getUser())); + + pipeline.setProjectId(source.getProjectId()); return pipeline; } diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/PipelineServiceImpl.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/PipelineServiceImpl.java index dcb81f9..7a04509 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/PipelineServiceImpl.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/impl/PipelineServiceImpl.java @@ -55,6 +55,14 @@ public class PipelineServiceImpl implements PipelineService { return newPipeline; } + @Override + @Transactional + public List createAll(@NonNull List newPipelines) { + return newPipelines.stream() + .map(this::create) + .collect(Collectors.toList()); + } + private void notifyNewPipeline(Pipeline pipeline, String oldStatus) { if (isNeedNotifyNewPipeline(pipeline)) { notifyService.send( diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java index f1088b4..ec40b77 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/MergeRequestParser.java @@ -9,27 +9,23 @@ import dev.struchkov.bot.gitlab.context.service.MergeRequestsService; import dev.struchkov.bot.gitlab.context.service.ProjectService; import dev.struchkov.bot.gitlab.core.config.properties.GitlabProperty; import dev.struchkov.bot.gitlab.core.config.properties.PersonProperty; -import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetMergeRequestFromGitlab; +import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetMergeRequestTask; +import dev.struchkov.bot.gitlab.core.utils.PoolUtils; import dev.struchkov.bot.gitlab.core.utils.StringUtils; import dev.struchkov.bot.gitlab.sdk.domain.CommitJson; import dev.struchkov.bot.gitlab.sdk.domain.MergeRequestJson; import dev.struchkov.haiti.utils.network.HttpParse; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.convert.ConversionService; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinTask; import java.util.stream.Collectors; @@ -41,7 +37,6 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; @Slf4j @Service -@RequiredArgsConstructor public class MergeRequestParser { private static final Set OLD_STATUSES = Set.of( @@ -54,16 +49,22 @@ public class MergeRequestParser { private final ConversionService conversionService; private final PersonProperty personProperty; - private ForkJoinPool forkJoinPool; + private final ForkJoinPool forkJoinPool; - @PreDestroy - public void preDestroy() { - forkJoinPool.shutdown(); - } - - @PostConstruct - public void postConstruct() { - forkJoinPool = new ForkJoinPool(4); + public MergeRequestParser( + GitlabProperty gitlabProperty, + MergeRequestsService mergeRequestsService, + ProjectService projectService, + ConversionService conversionService, + PersonProperty personProperty, + @Qualifier("parserPool") ForkJoinPool forkJoinPool + ) { + this.gitlabProperty = gitlabProperty; + this.mergeRequestsService = mergeRequestsService; + this.projectService = projectService; + this.conversionService = conversionService; + this.personProperty = personProperty; + this.forkJoinPool = forkJoinPool; } public void parsingOldMergeRequest() { @@ -129,30 +130,11 @@ public class MergeRequestParser { */ private List getMergeRequests(Set projectIds) { final List>> tasks = projectIds.stream() - .map(projectId -> new GetMergeRequestFromGitlab(projectId, gitlabProperty.getUrlPullRequestOpen(), personProperty.getToken())) + .map(projectId -> new GetMergeRequestTask(projectId, gitlabProperty.getUrlPullRequestOpen(), personProperty.getToken())) .map(forkJoinPool::submit) .collect(Collectors.toList()); - final List mergeRequestJsons = new ArrayList<>(); - Iterator>> iterator = tasks.iterator(); - while (!tasks.isEmpty()) { - while (iterator.hasNext()) { - final ForkJoinTask> task = iterator.next(); - if (task.isDone()) { - final List jsons; - try { - jsons = task.get(); - mergeRequestJsons.addAll(jsons); - } catch (InterruptedException | ExecutionException e) { - log.error(e.getMessage(), e); - Thread.currentThread().interrupt(); - } - iterator.remove(); - } - } - iterator = tasks.iterator(); - } - return mergeRequestJsons; + return PoolUtils.pullTaskResults(tasks); } private static void personMapping(List newMergeRequests) { diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/PipelineParser.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/PipelineParser.java index 79ff549..729dec3 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/PipelineParser.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/PipelineParser.java @@ -7,11 +7,14 @@ import dev.struchkov.bot.gitlab.context.service.PipelineService; import dev.struchkov.bot.gitlab.context.service.ProjectService; import dev.struchkov.bot.gitlab.core.config.properties.GitlabProperty; import dev.struchkov.bot.gitlab.core.config.properties.PersonProperty; +import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetPipelineShortTask; +import dev.struchkov.bot.gitlab.core.service.parser.forktask.GetPipelineTask; import dev.struchkov.bot.gitlab.core.utils.StringUtils; import dev.struchkov.bot.gitlab.sdk.domain.PipelineJson; +import dev.struchkov.bot.gitlab.sdk.domain.PipelineShortJson; import dev.struchkov.haiti.utils.network.HttpParse; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.convert.ConversionService; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -20,7 +23,10 @@ import org.springframework.stereotype.Service; import java.text.MessageFormat; import java.time.LocalDateTime; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; import java.util.stream.Collectors; import static dev.struchkov.bot.gitlab.context.domain.PipelineStatus.CREATED; @@ -29,6 +35,8 @@ import static dev.struchkov.bot.gitlab.context.domain.PipelineStatus.PENDING; import static dev.struchkov.bot.gitlab.context.domain.PipelineStatus.PREPARING; import static dev.struchkov.bot.gitlab.context.domain.PipelineStatus.RUNNING; import static dev.struchkov.bot.gitlab.context.domain.PipelineStatus.WAITING_FOR_RESOURCE; +import static dev.struchkov.bot.gitlab.core.utils.PoolUtils.pullTaskResult; +import static dev.struchkov.bot.gitlab.core.utils.PoolUtils.pullTaskResults; import static dev.struchkov.haiti.context.exception.ConvertException.convertException; import static dev.struchkov.haiti.utils.Checker.checkNotEmpty; import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; @@ -40,82 +48,94 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; */ @Slf4j @Service -@RequiredArgsConstructor public class PipelineParser { public static final Integer COUNT = 100; private static final Set oldStatus = Set.of( CREATED, WAITING_FOR_RESOURCE, PREPARING, PENDING, RUNNING, MANUAL ); + private final PipelineService pipelineService; private final ProjectService projectService; private final GitlabProperty gitlabProperty; private final PersonProperty personProperty; private final ConversionService conversionService; + private final ForkJoinPool forkJoinPool; private LocalDateTime lastUpdate = LocalDateTime.now(); + public PipelineParser( + PipelineService pipelineService, + ProjectService projectService, + GitlabProperty gitlabProperty, + PersonProperty personProperty, + ConversionService conversionService, + @Qualifier("parserPool") ForkJoinPool forkJoinPool + ) { + this.pipelineService = pipelineService; + this.projectService = projectService; + this.gitlabProperty = gitlabProperty; + this.personProperty = personProperty; + this.conversionService = conversionService; + this.forkJoinPool = forkJoinPool; + } + public void scanNewPipeline() { - log.debug("Старт обработки новых папйплайнов"); - int page = 0; + log.debug("Старт обработки новых пайплайнов"); final Set projectIds = projectService.getAllIds(); - for (Long projectId : projectIds) { - processingProject(projectId); - } + final Map pipelineProjectMap = getPipelineShortJsons(projectIds).stream() + .collect(Collectors.toMap(PipelineShortJson::getId, PipelineShortJson::getProjectId)); - log.debug("Конец обработки новых папйплайнов"); - } - - private void processingProject(Long projectId) { - int page = 1; - LocalDateTime newLastUpdate = LocalDateTime.now(); - List pipelineJsons = getPipelineJsons(projectId, page, lastUpdate); - - while (checkNotEmpty(pipelineJsons)) { - - final Set jsonIds = pipelineJsons.stream() - .map(PipelineJson::getId) - .collect(Collectors.toSet()); - - final ExistContainer existContainer = pipelineService.existsById(jsonIds); + if (!pipelineProjectMap.isEmpty()) { + final ExistContainer existContainer = pipelineService.existsById(pipelineProjectMap.keySet()); if (!existContainer.isAllFound()) { - final Set idsNotFound = existContainer.getIdNoFound(); - for (Long newId : idsNotFound) { - final Pipeline newPipeline = HttpParse.request( - MessageFormat.format(gitlabProperty.getUrlPipeline(), projectId, newId) - ) - .header(ACCEPT) - .header(StringUtils.H_PRIVATE_TOKEN, personProperty.getToken()) - .execute(PipelineJson.class) - .map(json -> { - final Pipeline pipeline = conversionService.convert(json, Pipeline.class); - pipeline.setProjectId(projectId); - return pipeline; - }) - .orElseThrow(convertException("Ошибка обновления Pipelines")); - pipelineService.create(newPipeline); - } + final List> tasks = idsNotFound.stream() + .map(pipelineId -> new GetPipelineTask( + gitlabProperty.getUrlPipeline(), + pipelineProjectMap.get(pipelineId), + pipelineId, + personProperty.getToken() + )) + .map(forkJoinPool::submit) + .collect(Collectors.toList()); + final List pipelineJsons = pullTaskResult(tasks); + if (checkNotEmpty(pipelineJsons)) { + final List newPipelines = pipelineJsons.stream() + .map(json -> conversionService.convert(json, Pipeline.class)) + .collect(Collectors.toList()); + pipelineService.createAll(newPipelines); + } } - pipelineJsons = getPipelineJsons(projectId, ++page, lastUpdate); } - lastUpdate = newLastUpdate; + log.debug("Конец обработки новых пайплайнов"); } - private List getPipelineJsons(Long projectId, int page, LocalDateTime afterUpdate) { - return HttpParse.request(MessageFormat.format(gitlabProperty.getUrlPipelines(), projectId, page)) - .header(ACCEPT) - .header(StringUtils.H_PRIVATE_TOKEN, personProperty.getToken()) - .getParameter("updated_after", afterUpdate.minusHours(12L).toString()) - .executeList(PipelineJson.class); + private List getPipelineShortJsons(Set projectIds) { + LocalDateTime newLastUpdate = LocalDateTime.now(); + final List>> tasks = projectIds.stream() + .map(projectId -> new GetPipelineShortTask( + gitlabProperty.getUrlPipelines(), + projectId, + lastUpdate, + personProperty.getToken() + )) + .map(forkJoinPool::submit) + .collect(Collectors.toList()); + + final List pipelineJsons = pullTaskResults(tasks); + + lastUpdate = newLastUpdate; + return pipelineJsons; } + public void scanOldPipeline() { log.debug("Старт обработки старых папйплайнов"); int page = 0; diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestTask.java similarity index 65% rename from bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java rename to bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestTask.java index 7c460a5..c813f64 100644 --- a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestFromGitlab.java +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetMergeRequestTask.java @@ -16,30 +16,29 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; @Slf4j @AllArgsConstructor @RequiredArgsConstructor -public class GetMergeRequestFromGitlab extends RecursiveTask> { +public class GetMergeRequestTask extends RecursiveTask> { private static final int PAGE_COUNT = 100; private final long projectId; private int pageNumber = 0; private final String urlMrOpen; - private final String personToken; + private final String gitlabToken; @Override protected List compute() { - final List mergeRequestJsons = getMergeRequestJsons(urlMrOpen, projectId, pageNumber, personToken); + final List mergeRequestJsons = getMergeRequestJsons(); if (mergeRequestJsons.size() == PAGE_COUNT) { - final GetMergeRequestFromGitlab newTask = new GetMergeRequestFromGitlab(projectId, pageNumber + 1, urlMrOpen, personToken); + final GetMergeRequestTask newTask = new GetMergeRequestTask(projectId, pageNumber + 1, urlMrOpen, gitlabToken); newTask.fork(); - final List result = newTask.join(); - mergeRequestJsons.addAll(result); + mergeRequestJsons.addAll(newTask.join()); } return mergeRequestJsons; } - private List getMergeRequestJsons(String url, Long projectId, int page, String personToken) { - final List jsons = HttpParse.request(MessageFormat.format(url, projectId, page, PAGE_COUNT)) - .header(StringUtils.H_PRIVATE_TOKEN, personToken) + private List getMergeRequestJsons() { + final List jsons = HttpParse.request(MessageFormat.format(urlMrOpen, projectId, pageNumber, PAGE_COUNT)) + .header(StringUtils.H_PRIVATE_TOKEN, gitlabToken) .header(ACCEPT) .executeList(MergeRequestJson.class); log.trace("Получено {} шт потенциально новых MR для проекта id:'{}' ", jsons.size(), projectId); diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineShortTask.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineShortTask.java new file mode 100644 index 0000000..4f2fadb --- /dev/null +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineShortTask.java @@ -0,0 +1,51 @@ +package dev.struchkov.bot.gitlab.core.service.parser.forktask; + +import dev.struchkov.bot.gitlab.core.utils.StringUtils; +import dev.struchkov.bot.gitlab.sdk.domain.PipelineShortJson; +import dev.struchkov.haiti.utils.network.HttpParse; +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.RecursiveTask; + +import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; + +@Slf4j +@AllArgsConstructor +@RequiredArgsConstructor +public class GetPipelineShortTask extends RecursiveTask> { + + private static final int PAGE_COUNT = 100; + + private final String urlPipelines; + private final long projectId; + private int pageNumber = 0; + private final LocalDateTime lastUpdate; + private final String gitlabToken; + + @Override + protected List compute() { + final List jsons = getPipelineJsons(); + if (jsons.size() == PAGE_COUNT) { + final GetPipelineShortTask newTask = new GetPipelineShortTask(urlPipelines, projectId, pageNumber + 1, lastUpdate, gitlabToken); + newTask.fork(); + jsons.addAll(newTask.join()); + } + jsons.forEach(pipelineJson -> pipelineJson.setProjectId(projectId)); + return jsons; + } + + private List getPipelineJsons() { + final List jsons = HttpParse.request(MessageFormat.format(urlPipelines, projectId, pageNumber, PAGE_COUNT)) + .header(ACCEPT) + .header(StringUtils.H_PRIVATE_TOKEN, gitlabToken) + .getParameter("updated_after", lastUpdate.minusHours(12L).toString()) + .executeList(PipelineShortJson.class); + log.trace("Получено {} шт потенциально новых пайплайнов для проекта id:'{}' ", jsons.size(), projectId); + return jsons; + } +} diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineTask.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineTask.java new file mode 100644 index 0000000..7c9b415 --- /dev/null +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineTask.java @@ -0,0 +1,37 @@ +package dev.struchkov.bot.gitlab.core.service.parser.forktask; + +import dev.struchkov.bot.gitlab.core.utils.StringUtils; +import dev.struchkov.bot.gitlab.sdk.domain.PipelineJson; +import dev.struchkov.haiti.utils.network.HttpParse; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.text.MessageFormat; +import java.util.concurrent.RecursiveTask; + +import static dev.struchkov.haiti.context.exception.ConvertException.convertException; +import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT; + +@Slf4j +@RequiredArgsConstructor +public class GetPipelineTask extends RecursiveTask { + + private final String urlPipeline; + private final long projectId; + private final long pipelineId; + private final String gitlabToken; + + @Override + protected PipelineJson compute() { + final PipelineJson pipelineJson = HttpParse.request( + MessageFormat.format(urlPipeline, projectId, pipelineId) + ) + .header(ACCEPT) + .header(StringUtils.H_PRIVATE_TOKEN, gitlabToken) + .execute(PipelineJson.class) + .orElseThrow(convertException("Ошибка обновления Pipelines")); + pipelineJson.setProjectId(projectId); + return pipelineJson; + } + +} diff --git a/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/utils/PoolUtils.java b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/utils/PoolUtils.java new file mode 100644 index 0000000..e5caeec --- /dev/null +++ b/bot-core/src/main/java/dev/struchkov/bot/gitlab/core/utils/PoolUtils.java @@ -0,0 +1,60 @@ +package dev.struchkov.bot.gitlab.core.utils; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinTask; + +@Slf4j +@UtilityClass +public class PoolUtils { + + public static List pullTaskResults(List>> tasks) { + final List results = new ArrayList<>(); + Iterator>> iterator = tasks.iterator(); + while (!tasks.isEmpty()) { + while (iterator.hasNext()) { + final ForkJoinTask> task = iterator.next(); + if (task.isDone()) { + final List jsons; + try { + jsons = task.get(); + results.addAll(jsons); + } catch (InterruptedException | ExecutionException e) { + log.error(e.getMessage(), e); + Thread.currentThread().interrupt(); + } + iterator.remove(); + } + } + iterator = tasks.iterator(); + } + return results; + } + + public static List pullTaskResult(List> tasks) { + final List results = new ArrayList<>(); + Iterator> iterator = tasks.iterator(); + while (!tasks.isEmpty()) { + while (iterator.hasNext()) { + final ForkJoinTask task = iterator.next(); + if (task.isDone()) { + try { + results.add(task.get()); + } catch (InterruptedException | ExecutionException e) { + log.error(e.getMessage(), e); + Thread.currentThread().interrupt(); + } + iterator.remove(); + } + } + iterator = tasks.iterator(); + } + return results; + } + +} diff --git a/gitlab-app/src/main/resources/application.yml b/gitlab-app/src/main/resources/application.yml index 8aeb28f..59e517f 100644 --- a/gitlab-app/src/main/resources/application.yml +++ b/gitlab-app/src/main/resources/application.yml @@ -46,7 +46,7 @@ gitlab-bot: users-url: ${GITLAB_URL}/api/v4/users url-note: "{0}#note_{1,number,#}" url-note-api: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/notes/{2,number,#}" - url-pipelines: "${GITLAB_URL}/api/v4/projects/{0,number,#}/pipelines?&page={1,number,#}&per_page=100" + url-pipelines: "${GITLAB_URL}/api/v4/projects/{0,number,#}/pipelines?&page={1,number,#}&per_page={2,number,#}" url-pipeline: "${GITLAB_URL}/api/v4/projects/{0,number,#}/pipelines/{1,number,#}" url-commit: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/commits?&page=1&per_page=1" url-new-note: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}/notes?body={3}" diff --git a/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineJson.java b/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineJson.java index 3ad3809..f1d06cf 100644 --- a/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineJson.java +++ b/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineJson.java @@ -36,4 +36,8 @@ public class PipelineJson { @JsonProperty("web_url") private String webUrl; + // Поля ниже не отдаются гитлабом, а заполняются вручную + + private Long projectId; + } diff --git a/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineShortJson.java b/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineShortJson.java new file mode 100644 index 0000000..74e22c7 --- /dev/null +++ b/gitlab-sdk/src/main/java/dev/struchkov/bot/gitlab/sdk/domain/PipelineShortJson.java @@ -0,0 +1,41 @@ +package dev.struchkov.bot.gitlab.sdk.domain; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * @author upagge 17.01.2021 + */ +@Data +public class PipelineShortJson { + + private Long id; + + @JsonSerialize(using = LocalDateTimeSerializer.class) + @JsonDeserialize(using = LocalDateTimeDeserializer.class) + @JsonProperty("created_at") + private LocalDateTime created; + + @JsonSerialize(using = LocalDateTimeSerializer.class) + @JsonDeserialize(using = LocalDateTimeDeserializer.class) + @JsonProperty("updated_at") + private LocalDateTime updated; + + private PipelineStatusJson status; + + private String ref; + + @JsonProperty("web_url") + private String webUrl; + + // Поля ниже не отдаются гитлабом, а заполняются вручную + + private Long projectId; + +}