Telegram-Bots/gitlab-notification
4
1
Fork 0
Code Issues 21 Pull Requests 1 Packages Projects 2 Releases 1 Activity

Оптимизация парсинга дискуссий

Browse Source
This commit is contained in:
Struchkov Mark 2022-12-20 22:19:45 +03:00
parent 35e206a7e2
commit a611e46479
Signed by: upagge
GPG Key ID: D3018BE7BA428CA6
5 changed files with 18 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/DiscussionParser.java
View File

@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static dev.struchkov.bot.gitlab.core.utils.StringUtils.H_PRIVATE_TOKEN;
import static dev.struchkov.haiti.utils.Checker.checkFalse;
import static dev.struchkov.haiti.utils.Checker.checkNotEmpty;
import static dev.struchkov.haiti.utils.Checker.checkNotNull;
import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT;
@ -42,6 +43,8 @@ import static dev.struchkov.haiti.utils.network.HttpParse.ACCEPT;
@Component
public class DiscussionParser {
public static final int PAGE_COUNT = 100;
private final DiscussionService discussionService;
private final MergeRequestsService mergeRequestsService;
@ -75,22 +78,20 @@ public class DiscussionParser {
log.debug("Старт обработки новых дискуссий");
final List<MergeRequestForDiscussion> mergeRequests = mergeRequestsService.getAllForDiscussion();
for (MergeRequestForDiscussion mergeRequest : mergeRequests) {
processingMergeRequest(mergeRequest);
}
mergeRequests.forEach(this::processingNewDiscussion);
log.debug("Конец обработки новых дискуссий");
}
private void processingMergeRequest(MergeRequestForDiscussion mergeRequest) {
private void processingNewDiscussion(MergeRequestForDiscussion mergeRequest) {
int page = 1;
List<DiscussionJson> discussionJson = getDiscussionJson(mergeRequest, page);
while (!discussionJson.isEmpty()) {
final List<DiscussionJson> discussionJson = getDiscussionJson(mergeRequest, page);
if (checkNotEmpty(discussionJson)) {
while (discussionJson.size() == PAGE_COUNT) {
discussionJson.addAll(getDiscussionJson(mergeRequest, ++page));
}
createNewDiscussion(discussionJson, mergeRequest);
discussionJson = getDiscussionJson(mergeRequest, ++page);
}
}
@ -100,9 +101,10 @@ public class DiscussionParser {
.collect(Collectors.toUnmodifiableSet());
final ExistContainer<Discussion, String> existContainer = discussionService.existsById(discussionIds);
if (!existContainer.isAllFound()) {
final Set<String> notFoundIds = existContainer.getIdNoFound();
if (checkFalse(existContainer.isAllFound())) {
final List<Discussion> newDiscussions = discussionJson.stream()
.filter(json -> existContainer.getIdNoFound().contains(json.getId()))
.filter(json -> notFoundIds.contains(json.getId()))
.map(json -> {
final Discussion discussion = conversionService.convert(json, Discussion.class);
discussion.setMergeRequest(mergeRequest);
@ -201,7 +203,7 @@ public class DiscussionParser {
}
private List<DiscussionJson> getDiscussionJson(MergeRequestForDiscussion mergeRequest, int page) {
return HttpParse.request(MessageFormat.format(gitlabProperty.getDiscussionsUrl(), mergeRequest.getProjectId(), mergeRequest.getTwoId(), page))
return HttpParse.request(MessageFormat.format(gitlabProperty.getDiscussionsUrl(), mergeRequest.getProjectId(), mergeRequest.getTwoId(), page, PAGE_COUNT))
.header(ACCEPT)
.header(H_PRIVATE_TOKEN, personProperty.getToken())
.executeList(DiscussionJson.class);

2
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllDiscussionForMergeRequestTask.java
View File

@ -24,7 +24,7 @@ public class GetAllDiscussionForMergeRequestTask extends RecursiveTask<List<Disc
private final long projectId;
private final long mergeRequestTwoId;
private final String personalGitlabToken;
private int page = 0;
private int page = 1;
@Override
@SneakyThrows

2
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetAllMergeRequestForProjectTask.java
View File

@ -23,7 +23,7 @@ public class GetAllMergeRequestForProjectTask extends RecursiveTask<List<MergeRe
private static final int PAGE_COUNT = 100;
private final long projectId;
private int pageNumber = 0;
private int pageNumber = 1;
private final String urlMrOpen;
private final String gitlabToken;

2
bot-core/src/main/java/dev/struchkov/bot/gitlab/core/service/parser/forktask/GetPipelineShortTask.java
View File

@ -24,7 +24,7 @@ public class GetPipelineShortTask extends RecursiveTask<List<PipelineShortJson>>
private final String urlPipelines;
private final long projectId;
private int pageNumber = 0;
private int pageNumber = 1;
private final LocalDateTime lastUpdate;
private final String gitlabToken;

2
gitlab-app/src/main/resources/application.yml
View File

@ -49,5 +49,5 @@ gitlab-bot:
pipeline-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/pipelines/{1,number,#}"
last-commit-of-merge-request-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/commits?&page=1&per_page=1"
new-note-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}/notes?body={3}"
discussions-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions?&page={2,number,#}&per_page=100"
discussions-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions?&page={2,number,#}&per_page={3,number,#}"
discussion-url: "${GITLAB_URL}/api/v4/projects/{0,number,#}/merge_requests/{1,number,#}/discussions/{2}"