Оптимизация парсинга дискуссий

bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/DiscussionParser.java

@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static dev.struchkov.bot.gitlab.core.utils.StringUtils.H_PRIVATE_TOKEN;
+import static dev.struchkov.haiti.utils.Checker.checkFalse;
 import static dev.struchkov.haiti.utils.Checker.checkNotEmpty;
 import static dev.struchkov.haiti.utils.Checker.checkNotNull;
 import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT;
@@ -42,6 +43,8 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT;
 @Component
 public class DiscussionParser {
 
+    public static final int PAGE_COUNT = 100;
+
     private final DiscussionService discussionService;
 
     private final MergeRequestsService mergeRequestsService;
@@ -75,22 +78,20 @@ public class DiscussionParser {
         log.debug("Старт обработки новых дискуссий");
         final List<MergeRequestForDiscussion> mergeRequests = mergeRequestsService.getAllForDiscussion();
 
-        for (MergeRequestForDiscussion mergeRequest : mergeRequests) {
+        mergeRequests.forEach(this::processingNewDiscussion);
-            processingMergeRequest(mergeRequest);
-        }
 
         log.debug("Конец обработки новых дискуссий");
     }
 
-    private void processingMergeRequest(MergeRequestForDiscussion mergeRequest) {
+    private void processingNewDiscussion(MergeRequestForDiscussion mergeRequest) {
         int page = 1;
-        List<DiscussionJson> discussionJson = getDiscussionJson(mergeRequest, page);
+        final List<DiscussionJson> discussionJson = getDiscussionJson(mergeRequest, page);
-
-        while (!discussionJson.isEmpty()) {
 
+        if (checkNotEmpty(discussionJson)) {
+            while (discussionJson.size() == PAGE_COUNT) {
+                discussionJson.addAll(getDiscussionJson(mergeRequest, ++page));
+            }
             createNewDiscussion(discussionJson, mergeRequest);
-
-            discussionJson = getDiscussionJson(mergeRequest, ++page);
         }
     }
 
@@ -100,9 +101,10 @@ public class DiscussionParser {
                 .collect(Collectors.toUnmodifiableSet());
 
         final ExistContainer<Discussion, String> existContainer = discussionService.existsById(discussionIds);
-        if (!existContainer.isAllFound()) {
+        final Set<String> notFoundIds = existContainer.getIdNoFound();
+        if (checkFalse(existContainer.isAllFound())) {
             final List<Discussion> newDiscussions = discussionJson.stream()
-                    .filter(json -> existContainer.getIdNoFound().contains(json.getId()))
+                    .filter(json -> notFoundIds.contains(json.getId()))
                     .map(json -> {
                         final Discussion discussion = conversionService.convert(json, Discussion.class);
                         discussion.setMergeRequest(mergeRequest);
@@ -201,7 +203,7 @@ public class DiscussionParser {
     }
 
     private List<DiscussionJson> getDiscussionJson(MergeRequestForDiscussion mergeRequest, int page) {
-        return HttpParse.request(MessageFormat.format(gitlabProperty.getDiscussionsUrl(), mergeRequest.getProjectId(), mergeRequest.getTwoId(), page))
+        return HttpParse.request(MessageFormat.format(gitlabProperty.getDiscussionsUrl(), mergeRequest.getProjectId(), mergeRequest.getTwoId(), page, PAGE_COUNT))
                 .header(ACCEPT)
                 .header(H_PRIVATE_TOKEN, personProperty.getToken())
                 .executeList(DiscussionJson.class);

bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllDiscussionForMergeRequestTask.java

@@ -24,7 +24,7 @@ public class GetAllDiscussionForMergeRequestTask extends RecursiveTask<List<Disc
     private final long projectId;
     private final long mergeRequestTwoId;
     private final String personalGitlabToken;
-    private int page = 0;
+    private int page = 1;
 
     @Override
     @SneakyThrows

bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllMergeRequestForProjectTask.java

@@ -23,7 +23,7 @@ public class GetAllMergeRequestForProjectTask extends RecursiveTask<List<MergeRe
     private static final int PAGE_COUNT = 100;
 
     private final long projectId;
-    private int pageNumber = 0;
+    private int pageNumber = 1;
     private final String urlMrOpen;
     private final String gitlabToken;
 

bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineShortTask.java

@@ -24,7 +24,7 @@ public class GetPipelineShortTask extends RecursiveTask<List<PipelineShortJson>>
 
     private final String urlPipelines;
     private final long projectId;
-    private int pageNumber = 0;
+    private int pageNumber = 1;
     private final LocalDateTime lastUpdate;
     private final String gitlabToken;
 

gitlab-app/src/main/resources/application.yml

@@ -49,5 +49,5 @@ gitlab-bot:
     pipeline-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/pipelines/{1,number,#}"
     last-commit-of-merge-request-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/commits?&page=1&per_page=1"
     new-note-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}/notes?body={3}"
-    discussions-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions?&page={2,number,#}&per_page=100"
+    discussions-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions?&page={2,number,#}&per_page={3,number,#}"
     discussion-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}"