From 7b2b4a90f1f5a2ef42e38545c72862d0dfe85ca0 Mon Sep 17 00:00:00 2001 From: Charishma0249 Date: Tue, 23 Nov 2021 14:00:56 -0700 Subject: [PATCH 01/49] [CITE-163] Add Giles Implementation --- citesphere-importer/pom.xml | 14 +- .../service/giles/GilesUploadService.java | 11 ++ .../giles/impl/GilesUploadServiceImpl.java | 97 ++++++++++++++ .../service/giles/impl/UploadResponse.java | 22 ++++ .../service/oauth/InternalTokenManager.java | 11 ++ .../core/service/oauth/OAuthScope.java | 16 +++ .../oauth/impl/InternalTokenManagerImpl.java | 120 ++++++++++++++++++ 7 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java diff --git a/citesphere-importer/pom.xml b/citesphere-importer/pom.xml index 4d1ee8d..db04f3a 100644 --- a/citesphere-importer/pom.xml +++ b/citesphere-importer/pom.xml @@ -16,6 +16,8 @@ 3.0.11.RELEASE 2.2.6.RELEASE 0.2 + 1.12 + 2.4.0.RELEASE $2a$04$oQo44vqcDIFRoYKiAXoNheurzkwX9dcNmowvTX/hsWuBMwijqn44i @@ -69,7 +71,11 @@ citesphere-messages ${citesphere.messages.version} - + + edu.asu.diging + citesphere-model + ${citesphere.model.version} + org.springframework @@ -122,6 +128,12 @@ spring-aop + + org.springframework.security.oauth + spring-security-oauth2 + ${spring-security-oauth2-version} + + org.aspectj aspectjrt diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java new file mode 100644 index 0000000..ef0f484 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java @@ -0,0 +1,11 @@ +package edu.asu.diging.citesphere.importer.core.service.giles; + +import org.springframework.web.multipart.MultipartFile; + +import edu.asu.diging.citesphere.model.bib.IGilesUpload; +import edu.asu.diging.citesphere.user.IUser; + +public interface GilesUploadService { + + IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes); +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java new file mode 100644 index 0000000..e63163c --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java @@ -0,0 +1,97 @@ +package edu.asu.diging.citesphere.importer.core.service.giles.impl; + +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import edu.asu.diging.citesphere.importer.core.service.giles.impl.UploadResponse; +import edu.asu.diging.citesphere.importer.core.service.giles.impl.GilesUploadServiceImpl.MultipartFileResource; +import edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager; +import edu.asu.diging.citesphere.importer.core.service.giles.GilesUploadService; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; +import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; +import edu.asu.diging.citesphere.user.IUser; + +public class GilesUploadServiceImpl implements GilesUploadService{ + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private RestTemplate restTemplate; + + @Autowired + private InternalTokenManager internalTokenManager; + + @Value("${giles_baseurl}") + private String gilesBaseurl; + + @Value("${giles_upload_endpoint}") + private String uploadEndpoint; + + @PostConstruct + public void init() { + restTemplate = new RestTemplate(); + } + + /* + * (non-Javadoc) + * + * @see edu.asu.diging.citesphere.core.service.giles.impl.GilesUploadService# + * uploadFile(edu.asu.diging.citesphere.user.IUser, + * org.springframework.web.multipart.MultipartFile, byte[]) + */ + @Override + public IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes) { + + String token = getToken(user); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.setBearerAuth(token); + + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("files", new MultipartFileResource(fileBytes, file.getOriginalFilename())); + + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + + ResponseEntity response = restTemplate.postForEntity(gilesBaseurl + uploadEndpoint, requestEntity, UploadResponse.class); + + IGilesUpload upload = new GilesUpload(); + upload.setProgressId(response.getBody().getId()); + upload.setUploadingUser(user.getUsername()); + return upload; + } + + private String getToken(IUser user) { + OAuth2AccessToken token = internalTokenManager.getAccessToken(user); + return token.getValue(); + } + + public class MultipartFileResource extends ByteArrayResource { + + private String filename; + + public MultipartFileResource(byte[] bytearray, String filename) { + super(bytearray); + this.filename = filename; + } + + @Override + public String getFilename() { + return this.filename; + } + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java new file mode 100644 index 0000000..76fe9c8 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/UploadResponse.java @@ -0,0 +1,22 @@ +package edu.asu.diging.citesphere.importer.core.service.giles.impl; + +class UploadResponse { + private String id; + private String checkUrl; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCheckUrl() { + return checkUrl; + } + + public void setCheckUrl(String checkUrl) { + this.checkUrl = checkUrl; + } +} \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java new file mode 100644 index 0000000..713fc16 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java @@ -0,0 +1,11 @@ +package edu.asu.diging.citesphere.importer.core.service.oauth; + +import org.springframework.security.oauth2.common.OAuth2AccessToken; + +import edu.asu.diging.citesphere.user.IUser; + +public interface InternalTokenManager { + + OAuth2AccessToken getAccessToken(IUser user); + +} \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java new file mode 100644 index 0000000..db8a49c --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java @@ -0,0 +1,16 @@ +package edu.asu.diging.citesphere.importer.core.service.oauth; + +public enum OAuthScope { + READ("read"), + WRITE("write"); + + private String scope; + + private OAuthScope(String scope) { + this.scope = scope; + } + + public String getScope() { + return scope; + } +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java new file mode 100644 index 0000000..2ae0999 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java @@ -0,0 +1,120 @@ +package edu.asu.diging.citesphere.importer.core.service.oauth.impl; + +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Date; +import java.util.Optional; +import java.util.UUID; + +import javax.annotation.PostConstruct; +import javax.transaction.Transactional; +import javax.transaction.Transactional.TxType; + +import org.javers.common.collections.Sets; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.common.DefaultOAuth2AccessToken; +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.provider.AuthorizationRequest; +import org.springframework.security.oauth2.provider.ClientDetails; +import org.springframework.security.oauth2.provider.ClientDetailsService; +import org.springframework.security.oauth2.provider.OAuth2Authentication; +import org.springframework.security.oauth2.provider.OAuth2Request; +import org.springframework.security.oauth2.provider.OAuth2RequestFactory; +import org.springframework.security.oauth2.provider.TokenRequest; +import org.springframework.security.oauth2.provider.implicit.ImplicitTokenRequest; +import org.springframework.security.oauth2.provider.request.DefaultOAuth2RequestFactory; +import org.springframework.security.oauth2.provider.token.TokenStore; +import org.springframework.stereotype.Service; + +import edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager; +import edu.asu.diging.citesphere.importer.core.service.oauth.OAuthScope; +import edu.asu.diging.citesphere.user.IUser; + +@Service +@PropertySource({ "classpath:config.properties", "${appConfigFile:classpath:}/app.properties" }) +@SuppressWarnings("deprecation") +@Transactional +public class InternalTokenManagerImpl implements InternalTokenManager { + + @Value("${_oauth_token_validity}") + private int tokenValidity; + + @Value("${_citephere_oauth2_app_clientid}") + private String citesphereClientId; + + @Autowired + private TokenStore tokenStore; + + @Autowired + private ClientDetailsService clientDetailsService; + + private OAuth2RequestFactory requestFactory; + + @PostConstruct + public void init() { + this.requestFactory = new DefaultOAuth2RequestFactory(clientDetailsService); + } + + /* (non-Javadoc) + * @see edu.asu.diging.citesphere.core.service.oauth.impl.InternalTokenManager#getAccessToken() + */ + @Override + @Transactional(TxType.REQUIRES_NEW) + public OAuth2AccessToken getAccessToken(IUser user) { + Collection tokens = tokenStore.findTokensByClientIdAndUserName(citesphereClientId, user.getUsername()); + Optional validToken = tokens.stream().filter(t -> !t.isExpired()).findFirst(); + if (validToken.isPresent()) { + return validToken.get(); + } + return createAccessToken(user); + } + + private OAuth2AccessToken createAccessToken(IUser user) { + DefaultOAuth2AccessToken token = new DefaultOAuth2AccessToken(UUID.randomUUID().toString()); + if (tokenValidity > 0) { + token.setExpiration(new Date(System.currentTimeMillis() + (tokenValidity * 1000L))); + } + token.setScope(Sets.asSet(OAuthScope.READ.getScope())); + + AuthorizationRequest request = new AuthorizationRequest(citesphereClientId, token.getScope()); + TokenRequest implicitRequest = new ImplicitTokenRequest(requestFactory.createTokenRequest(request, "implicit"), requestFactory.createOAuth2Request(request)); + + OAuth2Authentication authentication = getOAuth2Authentication(clientDetailsService.loadClientByClientId(citesphereClientId), implicitRequest, user); + tokenStore.storeAccessToken(token, authentication); + System.out.println(extractTokenKey(token.getValue())); + return token; + } + + private OAuth2Authentication getOAuth2Authentication(ClientDetails client, TokenRequest tokenRequest, IUser user) { + OAuth2Request storedOAuth2Request = requestFactory.createOAuth2Request(client, tokenRequest); + Authentication authentication = new UsernamePasswordAuthenticationToken(user, null, user.getRoles()); + return new OAuth2Authentication(storedOAuth2Request, authentication); + } + + private String extractTokenKey(String value) { + if(value == null) { + return null; + } else { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException var5) { + throw new IllegalStateException("MD5 algorithm not available. Fatal (should be in the JDK)."); + } + + try { + byte[] e = digest.digest(value.getBytes("UTF-8")); + return String.format("%032x", new Object[]{new BigInteger(1, e)}); + } catch (UnsupportedEncodingException var4) { + throw new IllegalStateException("UTF-8 encoding not available. Fatal (should be in the JDK)."); + } + } + } +} From ef6d57323b735d657792fe80e0a171d39d1cec08 Mon Sep 17 00:00:00 2001 From: Charishma0249 Date: Thu, 9 Jun 2022 19:39:59 -0400 Subject: [PATCH 02/49] [CITE-163] Giles Implementation --- .../giles/impl/GilesUploadServiceImpl.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java index e63163c..d642680 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java @@ -79,19 +79,4 @@ private String getToken(IUser user) { return token.getValue(); } - public class MultipartFileResource extends ByteArrayResource { - - private String filename; - - public MultipartFileResource(byte[] bytearray, String filename) { - super(bytearray); - this.filename = filename; - } - - @Override - public String getFilename() { - return this.filename; - } - } - } From afea37faccf736e50f51c39ce933cc086d11b572 Mon Sep 17 00:00:00 2001 From: Charishma0249 Date: Fri, 10 Jun 2022 10:58:26 -0400 Subject: [PATCH 03/49] [CITE-163] Giles Implementation --- .../giles/impl/GilesUploadServiceImpl.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java index d642680..e63163c 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java @@ -79,4 +79,19 @@ private String getToken(IUser user) { return token.getValue(); } + public class MultipartFileResource extends ByteArrayResource { + + private String filename; + + public MultipartFileResource(byte[] bytearray, String filename) { + super(bytearray); + this.filename = filename; + } + + @Override + public String getFilename() { + return this.filename; + } + } + } From 179f0b5f4e372964ceee32a71d18aeb2d6b764b4 Mon Sep 17 00:00:00 2001 From: Charishma0249 Date: Fri, 10 Jun 2022 19:35:55 -0400 Subject: [PATCH 04/49] [CITE-163] Adding OAuth properties --- .../src/main/resources/app.properties | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 citesphere-importer/src/main/resources/app.properties diff --git a/citesphere-importer/src/main/resources/app.properties b/citesphere-importer/src/main/resources/app.properties new file mode 100644 index 0000000..d975229 --- /dev/null +++ b/citesphere-importer/src/main/resources/app.properties @@ -0,0 +1,17 @@ +app.url=${app.url} +app.name=${app.name} +app.createUserApprovalPath=login/reset/initiated +app.resetPasswordPath= +app.resetPasswordSubject=Password Reset request for $app +password.reset.expiration=60 + +mongo.database.name=${mongo.database.name} +mongo.database.host=${mongo.database.host} +mongo.database.port=${mongo.database.port} +mongo.database.user=${mongo.database.user} +mongo.database.password=${mongo.database.password} +mongo.database.authdb=${mongo.database.authdb} + +_citephere_oauth2_app_clientid=${citesphere.oauth2.app.clientId} + +giles_baseurl=${giles.baseurl} \ No newline at end of file From 3a562d531515c3b4fa9e95a12a75a074ba4a0dc2 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 27 Aug 2024 09:47:05 -0700 Subject: [PATCH 05/49] [CITE-163] Added collection import kafka topic --- .../kafka/impl/ReferenceImportListener.java | 26 +++++++++++++++++-- .../impl/CollectionImportProcessor.java | 14 ++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java index 6b4f845..82f7e92 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.kafka.annotation.KafkaListener; import com.fasterxml.jackson.databind.ObjectMapper; @@ -18,7 +19,12 @@ public class ReferenceImportListener { private final Logger logger = LoggerFactory.getLogger(getClass()); @Autowired - private IImportProcessor processor; + @Qualifier("importProcessor") + private IImportProcessor fileProcessor; + + @Autowired + @Qualifier("collectionImportProcessor") + private IImportProcessor collectionProcessor; @KafkaListener(topics = KafkaTopics.REFERENCES_IMPORT_TOPIC) public void receiveMessage(String message) { @@ -32,6 +38,22 @@ public void receiveMessage(String message) { return; } - processor.process(msg); + fileProcessor.process(msg); } + + @KafkaListener(topics = KafkaTopics.COLLECTION_IMPORT_TOPIC) + public void receiveCollectionImportMessage(String message) { + ObjectMapper mapper = new ObjectMapper(); + KafkaJobMessage msg = null; + try { + msg = mapper.readValue(message, KafkaJobMessage.class); + } catch (IOException e) { + logger.error("Could not unmarshall message.", e); + // FIXME: handle this case + return; + } + + collectionProcessor.process(msg); + } + } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java new file mode 100644 index 0000000..1c2f1b0 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -0,0 +1,14 @@ +package edu.asu.diging.citesphere.importer.core.service.impl; + +import edu.asu.diging.citesphere.importer.core.service.IImportProcessor; +import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; + +public class CollectionImportProcessor implements IImportProcessor { + + @Override + public void process(KafkaJobMessage message) { + // TODO Auto-generated method stub + + } + +} From 6a61a1fefbcdfc3b4c8cce387104827f00a8614d Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 29 Aug 2024 12:06:11 -0700 Subject: [PATCH 06/49] [CITE-163] Added supporting methods to process the file --- .../impl/CollectionImportProcessor.java | 190 +++++++++++++++++- 1 file changed, 189 insertions(+), 1 deletion(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index 1c2f1b0..e9d79b3 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -1,14 +1,202 @@ package edu.asu.diging.citesphere.importer.core.service.impl; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; +import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; +import edu.asu.diging.citesphere.importer.core.exception.MessageCreationException; +import edu.asu.diging.citesphere.importer.core.kafka.impl.KafkaRequestProducer; +import edu.asu.diging.citesphere.importer.core.model.BibEntry; +import edu.asu.diging.citesphere.importer.core.model.ItemType; +import edu.asu.diging.citesphere.importer.core.model.impl.Publication; +import edu.asu.diging.citesphere.importer.core.service.ICitesphereConnector; import edu.asu.diging.citesphere.importer.core.service.IImportProcessor; +import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; +import edu.asu.diging.citesphere.importer.core.zotero.IZoteroConnector; +import edu.asu.diging.citesphere.importer.core.zotero.template.IJsonGenerationService; +import edu.asu.diging.citesphere.messages.KafkaTopics; +import edu.asu.diging.citesphere.messages.model.ItemCreationResponse; +import edu.asu.diging.citesphere.messages.model.KafkaImportReturnMessage; import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; +import edu.asu.diging.citesphere.messages.model.ResponseCode; +import edu.asu.diging.citesphere.messages.model.Status; +@Service public class CollectionImportProcessor implements IImportProcessor { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Autowired + private ICitesphereConnector connector; + + @Autowired + private IHandlerRegistry handlerRegistry; + + @Autowired + private IZoteroConnector zoteroConnector; + + @Autowired + private IJsonGenerationService generationService; + + @Autowired + private KafkaRequestProducer requestProducer; + + private Map itemTypeMapping = new HashMap<>(); + @PostConstruct + public void init() { + // this needs to be changed and improved, but for now it works + itemTypeMapping.put(Publication.ARTICLE, ItemType.JOURNAL_ARTICLE); + itemTypeMapping.put(Publication.BOOK, ItemType.BOOK); + itemTypeMapping.put(Publication.BOOK_CHAPTER, ItemType.BOOK_SECTION); + itemTypeMapping.put(Publication.LETTER, ItemType.LETTER); + itemTypeMapping.put(Publication.NEWS_ITEM, ItemType.NEWSPAPER_ARTICLE); + itemTypeMapping.put(Publication.PROCEEDINGS_PAPER, ItemType.CONFERENCE_PAPER); + itemTypeMapping.put(Publication.DOCUMENT, ItemType.DOCUMENT); + } + + /* + * (non-Javadoc) + * + * @see + * edu.asu.diging.citesphere.importer.core.service.impl.IImportProcessor#process + * (edu.asu.diging.citesphere.importer.core.kafka.impl.KafkaJobMessage) + */ @Override public void process(KafkaJobMessage message) { - // TODO Auto-generated method stub + JobInfo info = getJobInfo(message); + if (info == null) { + sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X10); + return; + } + + String filePath = downloadFile(message); + if (filePath == null) { + sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X20); + return; + } + + sendMessage(null, message.getId(), Status.PROCESSING, ResponseCode.P00); + BibEntryIterator bibIterator = null; +// try { +// bibIterator = handlerRegistry.handleFile(info, filePath); +// } catch (IteratorCreationException e1) { +// logger.error("Could not create iterator.", e1); +// } +// +// if (bibIterator == null) { +// sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X30); +// return; +// } + +// ObjectMapper mapper = new ObjectMapper(); +// ArrayNode root = mapper.createArrayNode(); +// int entryCounter = 0; +// while (bibIterator.hasNext()) { +// BibEntry entry = bibIterator.next(); +// if (entry.getArticleType() == null) { +// // something is wrong with this entry, let's ignore it +// continue; +// } +// ItemType type = itemTypeMapping.get(entry.getArticleType()); +// JsonNode template = zoteroConnector.getTemplate(type); +// ObjectNode bibNode = generationService.generateJson(template, entry); +// +// root.add(bibNode); +// entryCounter++; +// +// // we can submit max 50 entries to Zotoro +// if (entryCounter >= 50) { +// submitEntries(root, info); +// entryCounter = 0; +// root = mapper.createArrayNode(); +// } +// +// } +// +// bibIterator.close(); +// +// ItemCreationResponse response = null; +// if (entryCounter > 0) { +// response = submitEntries(root, info); +// } +// +// response = response != null ? response : new ItemCreationResponse(); +// sendMessage(response, message.getId(), Status.DONE, ResponseCode.S00); + + } + + private void sendMessage(ItemCreationResponse message, String jobId, Status status, ResponseCode code) { + KafkaImportReturnMessage returnMessage = new KafkaImportReturnMessage(message, jobId); + returnMessage.setStatus(status); + returnMessage.setCode(code); + try { + requestProducer.sendRequest(returnMessage, KafkaTopics.REFERENCES_IMPORT_DONE_TOPIC); + } catch (MessageCreationException e) { + // FIXME handle this case + logger.error("Exception sending message.", e); + } + } + + private ItemCreationResponse submitEntries(ArrayNode entries, JobInfo info) { + ObjectMapper mapper = new ObjectMapper(); + try { + String msg = mapper.writeValueAsString(entries); + logger.info("Submitting " + msg); + ItemCreationResponse response = zoteroConnector.addEntries(info, entries); + if (response != null) { + logger.info(response.getSuccessful() + ""); + logger.error(response.getFailed() + ""); + } else { + logger.error("Item creation failed."); + } + return response; + } catch (URISyntaxException e) { + logger.error("Could not store new entry.", e); + } catch (JsonProcessingException e) { + logger.error("Could not write JSON."); + } + return null; + } + + private JobInfo getJobInfo(KafkaJobMessage message) { + JobInfo info = null; + try { + info = connector.getJobInfo(message.getId()); + } catch (CitesphereCommunicationException e) { + // FIXME this needs to be handled better + logger.error("Could not get Zotero info.", e); + return null; + } + return info; + } + private String downloadFile(KafkaJobMessage message) { + String file = null; + try { + file = connector.getUploadeFile(message.getId()); + } catch (CitesphereCommunicationException e) { + // FIXME this needs to be handled better + logger.error("Could not get Zotero info.", e); + return null; + } + return file; } } From 45f600819832f073f58eb7652e88d0d2a79edfbd Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 30 Aug 2024 16:37:13 -0700 Subject: [PATCH 07/49] [CITE-163] Added code to process the collection import --- .../impl/CollectionImportProcessor.java | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index e9d79b3..3b9c67c 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -94,51 +94,51 @@ public void process(KafkaJobMessage message) { sendMessage(null, message.getId(), Status.PROCESSING, ResponseCode.P00); BibEntryIterator bibIterator = null; -// try { -// bibIterator = handlerRegistry.handleFile(info, filePath); -// } catch (IteratorCreationException e1) { -// logger.error("Could not create iterator.", e1); -// } -// -// if (bibIterator == null) { -// sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X30); -// return; -// } + try { + bibIterator = handlerRegistry.handleFile(info, filePath); + } catch (IteratorCreationException e1) { + logger.error("Could not create iterator.", e1); + } + + if (bibIterator == null) { + sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X30); + return; + } + + ObjectMapper mapper = new ObjectMapper(); + ArrayNode root = mapper.createArrayNode(); + int entryCounter = 0; + while (bibIterator.hasNext()) { + BibEntry entry = bibIterator.next(); + if (entry.getArticleType() == null) { + // something is wrong with this entry, let's ignore it + continue; + } + ItemType type = itemTypeMapping.get(entry.getArticleType()); + JsonNode template = zoteroConnector.getTemplate(type); + ObjectNode bibNode = generationService.generateJson(template, entry); + + root.add(bibNode); + entryCounter++; + + // we can submit max 50 entries to Zotoro + if (entryCounter >= 50) { + submitEntries(root, info); + entryCounter = 0; + root = mapper.createArrayNode(); + } + + } -// ObjectMapper mapper = new ObjectMapper(); -// ArrayNode root = mapper.createArrayNode(); -// int entryCounter = 0; -// while (bibIterator.hasNext()) { -// BibEntry entry = bibIterator.next(); -// if (entry.getArticleType() == null) { -// // something is wrong with this entry, let's ignore it -// continue; -// } -// ItemType type = itemTypeMapping.get(entry.getArticleType()); -// JsonNode template = zoteroConnector.getTemplate(type); -// ObjectNode bibNode = generationService.generateJson(template, entry); -// -// root.add(bibNode); -// entryCounter++; -// -// // we can submit max 50 entries to Zotoro -// if (entryCounter >= 50) { -// submitEntries(root, info); -// entryCounter = 0; -// root = mapper.createArrayNode(); -// } -// -// } -// -// bibIterator.close(); -// -// ItemCreationResponse response = null; -// if (entryCounter > 0) { -// response = submitEntries(root, info); -// } -// -// response = response != null ? response : new ItemCreationResponse(); -// sendMessage(response, message.getId(), Status.DONE, ResponseCode.S00); + bibIterator.close(); + + ItemCreationResponse response = null; + if (entryCounter > 0) { + response = submitEntries(root, info); + } + + response = response != null ? response : new ItemCreationResponse(); + sendMessage(response, message.getId(), Status.DONE, ResponseCode.S00); } From 73f3db31c437081c1643def33dbc18b380785bf7 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 27 Sep 2024 16:36:46 -0700 Subject: [PATCH 08/49] [CITE-163] Removed unwanted old code --- citesphere-importer/pom.xml | 2 +- .../oauth/impl/InternalTokenManagerImpl.java | 127 +++++++++--------- 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/citesphere-importer/pom.xml b/citesphere-importer/pom.xml index db04f3a..5c8d936 100644 --- a/citesphere-importer/pom.xml +++ b/citesphere-importer/pom.xml @@ -15,7 +15,7 @@ Lovelace-RELEASE 3.0.11.RELEASE 2.2.6.RELEASE - 0.2 + 0.5 1.12 2.4.0.RELEASE diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java index 2ae0999..52226da 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java @@ -43,24 +43,24 @@ @Transactional public class InternalTokenManagerImpl implements InternalTokenManager { - @Value("${_oauth_token_validity}") - private int tokenValidity; +// @Value("${_oauth_token_validity}") +// private Integer tokenValidity; +// +// @Value("${_citephere_oauth2_app_clientid}") +// private String citesphereClientId; +// +// @Autowired +// private TokenStore tokenStore; +// +// @Autowired +// private ClientDetailsService clientDetailsService; +// +// private OAuth2RequestFactory requestFactory; - @Value("${_citephere_oauth2_app_clientid}") - private String citesphereClientId; - - @Autowired - private TokenStore tokenStore; - - @Autowired - private ClientDetailsService clientDetailsService; - - private OAuth2RequestFactory requestFactory; - - @PostConstruct - public void init() { - this.requestFactory = new DefaultOAuth2RequestFactory(clientDetailsService); - } +// @PostConstruct +// public void init() { +// this.requestFactory = new DefaultOAuth2RequestFactory(clientDetailsService); +// } /* (non-Javadoc) * @see edu.asu.diging.citesphere.core.service.oauth.impl.InternalTokenManager#getAccessToken() @@ -68,53 +68,54 @@ public void init() { @Override @Transactional(TxType.REQUIRES_NEW) public OAuth2AccessToken getAccessToken(IUser user) { - Collection tokens = tokenStore.findTokensByClientIdAndUserName(citesphereClientId, user.getUsername()); - Optional validToken = tokens.stream().filter(t -> !t.isExpired()).findFirst(); - if (validToken.isPresent()) { - return validToken.get(); - } - return createAccessToken(user); +// Collection tokens = tokenStore.findTokensByClientIdAndUserName(citesphereClientId, user.getUsername()); +// Optional validToken = tokens.stream().filter(t -> !t.isExpired()).findFirst(); +// if (validToken.isPresent()) { +// return validToken.get(); +// } +// return createAccessToken(user); + return null; } - private OAuth2AccessToken createAccessToken(IUser user) { - DefaultOAuth2AccessToken token = new DefaultOAuth2AccessToken(UUID.randomUUID().toString()); - if (tokenValidity > 0) { - token.setExpiration(new Date(System.currentTimeMillis() + (tokenValidity * 1000L))); - } - token.setScope(Sets.asSet(OAuthScope.READ.getScope())); - - AuthorizationRequest request = new AuthorizationRequest(citesphereClientId, token.getScope()); - TokenRequest implicitRequest = new ImplicitTokenRequest(requestFactory.createTokenRequest(request, "implicit"), requestFactory.createOAuth2Request(request)); - - OAuth2Authentication authentication = getOAuth2Authentication(clientDetailsService.loadClientByClientId(citesphereClientId), implicitRequest, user); - tokenStore.storeAccessToken(token, authentication); - System.out.println(extractTokenKey(token.getValue())); - return token; - } - - private OAuth2Authentication getOAuth2Authentication(ClientDetails client, TokenRequest tokenRequest, IUser user) { - OAuth2Request storedOAuth2Request = requestFactory.createOAuth2Request(client, tokenRequest); - Authentication authentication = new UsernamePasswordAuthenticationToken(user, null, user.getRoles()); - return new OAuth2Authentication(storedOAuth2Request, authentication); - } +// private OAuth2AccessToken createAccessToken(IUser user) { +// DefaultOAuth2AccessToken token = new DefaultOAuth2AccessToken(UUID.randomUUID().toString()); +// if (tokenValidity > 0) { +// token.setExpiration(new Date(System.currentTimeMillis() + (tokenValidity * 1000L))); +// } +// token.setScope(Sets.asSet(OAuthScope.READ.getScope())); +// +// AuthorizationRequest request = new AuthorizationRequest(citesphereClientId, token.getScope()); +// TokenRequest implicitRequest = new ImplicitTokenRequest(requestFactory.createTokenRequest(request, "implicit"), requestFactory.createOAuth2Request(request)); +// +// OAuth2Authentication authentication = getOAuth2Authentication(clientDetailsService.loadClientByClientId(citesphereClientId), implicitRequest, user); +// tokenStore.storeAccessToken(token, authentication); +// System.out.println(extractTokenKey(token.getValue())); +// return token; +// } - private String extractTokenKey(String value) { - if(value == null) { - return null; - } else { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException var5) { - throw new IllegalStateException("MD5 algorithm not available. Fatal (should be in the JDK)."); - } - - try { - byte[] e = digest.digest(value.getBytes("UTF-8")); - return String.format("%032x", new Object[]{new BigInteger(1, e)}); - } catch (UnsupportedEncodingException var4) { - throw new IllegalStateException("UTF-8 encoding not available. Fatal (should be in the JDK)."); - } - } - } +// private OAuth2Authentication getOAuth2Authentication(ClientDetails client, TokenRequest tokenRequest, IUser user) { +// OAuth2Request storedOAuth2Request = requestFactory.createOAuth2Request(client, tokenRequest); +// Authentication authentication = new UsernamePasswordAuthenticationToken(user, null, user.getRoles()); +// return new OAuth2Authentication(storedOAuth2Request, authentication); +// } +// +// private String extractTokenKey(String value) { +// if(value == null) { +// return null; +// } else { +// MessageDigest digest; +// try { +// digest = MessageDigest.getInstance("MD5"); +// } catch (NoSuchAlgorithmException var5) { +// throw new IllegalStateException("MD5 algorithm not available. Fatal (should be in the JDK)."); +// } +// +// try { +// byte[] e = digest.digest(value.getBytes("UTF-8")); +// return String.format("%032x", new Object[]{new BigInteger(1, e)}); +// } catch (UnsupportedEncodingException var4) { +// throw new IllegalStateException("UTF-8 encoding not available. Fatal (should be in the JDK)."); +// } +// } +// } } From d97177e1ffe49169d14257cec6f153d12bc6a854 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 4 Oct 2024 16:35:52 -0700 Subject: [PATCH 09/49] [CITE-163] Added BibFileHandler --- .../service/parse/impl/BibFileHandler.java | 36 +++++++++++++++++++ .../parse/iterators/BibFileIterator.java | 36 +++++++++++++++++++ .../parse/jstor/xml/BibEntryParser.java | 5 +++ 3 files changed, 77 insertions(+) create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java new file mode 100644 index 0000000..6eb87d1 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -0,0 +1,36 @@ +package edu.asu.diging.citesphere.importer.core.service.parse.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; +import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; +import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; +import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; +import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; +import edu.asu.diging.citesphere.importer.core.service.parse.iterators.BibFileIterator; +import edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml.BibEntryParser; + +@Service +public class BibFileHandler implements FileHandler { + + @Autowired + private BibEntryParser bibEntryParser; + + @Override + public boolean canHandle(String path) throws HandlerTestException { + if (path.toLowerCase().endsWith(".bib")) { + return true; + } + return false; + } + + @Override + public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) + throws IteratorCreationException { + System.out.println("inside bib iterator ................."); + return new BibFileIterator(path, bibEntryParser); + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java new file mode 100644 index 0000000..6687696 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -0,0 +1,36 @@ +package edu.asu.diging.citesphere.importer.core.service.parse.iterators; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import edu.asu.diging.citesphere.importer.core.model.BibEntry; +import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml.BibEntryParser; + +public class BibFileIterator implements BibEntryIterator { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + public BibFileIterator(String filePath, BibEntryParser parser) { + //TODO: add logic + } + + @Override + public BibEntry next() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean hasNext() { + // TODO Auto-generated method stub + return false; + } + + @Override + public void close() { + // TODO Auto-generated method stub + + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java new file mode 100644 index 0000000..8b1a474 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java @@ -0,0 +1,5 @@ +package edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml; + +public class BibEntryParser { + +} From 6235fec0dbb0e38e29fb08fcf4e76bc466af97cd Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 8 Oct 2024 12:01:43 -0700 Subject: [PATCH 10/49] [CITE-163] Added Bib file iterator code --- .../service/parse/impl/BibFileHandler.java | 9 ++--- .../parse/iterators/BibFileIterator.java | 35 ++++++++++++++++--- .../parse/jstor/xml/BibEntryParser.java | 21 ++++++++++- .../parse/jstor/xml/IBibEntryParser.java | 14 ++++++++ 4 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index 6eb87d1..0d75296 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -1,6 +1,5 @@ package edu.asu.diging.citesphere.importer.core.service.parse.impl; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; @@ -10,14 +9,10 @@ import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; import edu.asu.diging.citesphere.importer.core.service.parse.iterators.BibFileIterator; -import edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml.BibEntryParser; @Service public class BibFileHandler implements FileHandler { - @Autowired - private BibEntryParser bibEntryParser; - @Override public boolean canHandle(String path) throws HandlerTestException { if (path.toLowerCase().endsWith(".bib")) { @@ -29,8 +24,8 @@ public boolean canHandle(String path) throws HandlerTestException { @Override public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) throws IteratorCreationException { - System.out.println("inside bib iterator ................."); - return new BibFileIterator(path, bibEntryParser); + System.out.println("inside bib file handler ................."); + return new BibFileIterator(path); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 6687696..7c6281c 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -1,23 +1,50 @@ package edu.asu.diging.citesphere.importer.core.service.parse.iterators; +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.LineIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; -import edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml.BibEntryParser; public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); - public BibFileIterator(String filePath, BibEntryParser parser) { - //TODO: add logic + private String filePath; + private LineIterator lineIterator; + private String currentLine = null; + + public BibFileIterator(String filePath) { + System.out.println("------------------ inside bib file iterator"); + this.filePath = filePath; + init(); + } + + private void init() { + try { + lineIterator = FileUtils.lineIterator(new File(filePath)); + if (lineIterator.hasNext()) { + // we're at the beginning, so we'll signal that with and empty string + currentLine = ""; + String line = lineIterator.nextLine(); + System.out.println(line + "-------------------------------"); + } + } catch (IOException e) { + logger.error("Could not create line iterator.", e); + } } @Override public BibEntry next() { - // TODO Auto-generated method stub + while (lineIterator.hasNext()) { + String line = lineIterator.nextLine(); + System.out.println(line + " 2 -------------------------------"); + } return null; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java index 8b1a474..9c02fd2 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java @@ -1,5 +1,24 @@ package edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml; -public class BibEntryParser { +import org.springframework.stereotype.Component; +import org.w3c.dom.Node; + +import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; + +@Component +public class BibEntryParser implements IBibEntryParser { + + @Override + public void parseJournalMetaTag(Node node, ContainerMeta meta) { + // TODO Auto-generated method stub + + } + + @Override + public void parseArticleMetaTag(Node node, ArticleMeta meta) { + // TODO Auto-generated method stub + + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java new file mode 100644 index 0000000..6e0878a --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java @@ -0,0 +1,14 @@ +package edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml; + +import org.w3c.dom.Node; + +import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; + +public interface IBibEntryParser { + + void parseJournalMetaTag(Node node, ContainerMeta meta); + + void parseArticleMetaTag(Node node, ArticleMeta meta); + +} From 264a5ba9ab29f3138a48540158cd7316e253684d Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Wed, 9 Oct 2024 13:06:55 -0700 Subject: [PATCH 11/49] [CITE-163] Updated BibFile Iterator --- .../parse/iterators/BibFileIterator.java | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 7c6281c..5db336c 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -1,9 +1,10 @@ package edu.asu.diging.citesphere.importer.core.service.parse.iterators; -import java.io.File; +import java.io.BufferedReader; +import java.io.FileReader; import java.io.IOException; +import java.util.Iterator; -import org.apache.commons.io.FileUtils; import org.apache.commons.io.LineIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +17,7 @@ public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); private String filePath; - private LineIterator lineIterator; + private Iterator lineIterator; private String currentLine = null; public BibFileIterator(String filePath) { @@ -26,14 +27,24 @@ public BibFileIterator(String filePath) { } private void init() { - try { - lineIterator = FileUtils.lineIterator(new File(filePath)); - if (lineIterator.hasNext()) { - // we're at the beginning, so we'll signal that with and empty string - currentLine = ""; - String line = lineIterator.nextLine(); - System.out.println(line + "-------------------------------"); - } + // try { + // System.out.println(filePath + " filepath ======================"); + // lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); + // if (lineIterator.hasNext()) { + // // we're at the beginning, so we'll signal that with and empty string + //// currentLine = ""; + // String line = lineIterator.nextLine(); + // System.out.println(line + "-------------------------------"); + // } + + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + lineIterator = reader.lines().iterator(); + + // while (iterator.hasNext()) { + // String line = iterator.next(); + // // Process the line + // System.out.println(line); + // } } catch (IOException e) { logger.error("Could not create line iterator.", e); } @@ -42,7 +53,7 @@ private void init() { @Override public BibEntry next() { while (lineIterator.hasNext()) { - String line = lineIterator.nextLine(); + String line = lineIterator.next(); System.out.println(line + " 2 -------------------------------"); } return null; @@ -50,8 +61,7 @@ public BibEntry next() { @Override public boolean hasNext() { - // TODO Auto-generated method stub - return false; + return lineIterator.hasNext(); } @Override From c05de5cb7aa5280f84fc7b7cb712fce7f8204171 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 10 Oct 2024 12:05:46 -0700 Subject: [PATCH 12/49] [CITE-163] Parsed article Type of bib file --- .../parse/iterators/BibFileIterator.java | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 5db336c..1720226 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -1,60 +1,52 @@ package edu.asu.diging.citesphere.importer.core.service.parse.iterators; -import java.io.BufferedReader; -import java.io.FileReader; +import java.io.File; import java.io.IOException; import java.util.Iterator; -import org.apache.commons.io.LineIterator; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; +import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); - + private String filePath; private Iterator lineIterator; private String currentLine = null; - + public BibFileIterator(String filePath) { System.out.println("------------------ inside bib file iterator"); this.filePath = filePath; init(); } - - private void init() { - // try { - // System.out.println(filePath + " filepath ======================"); - // lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); - // if (lineIterator.hasNext()) { - // // we're at the beginning, so we'll signal that with and empty string - //// currentLine = ""; - // String line = lineIterator.nextLine(); - // System.out.println(line + "-------------------------------"); - // } - try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - lineIterator = reader.lines().iterator(); - - // while (iterator.hasNext()) { - // String line = iterator.next(); - // // Process the line - // System.out.println(line); - // } + private void init() { + try { + System.out.println(filePath + " filepath ======================"); + lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); } catch (IOException e) { logger.error("Could not create line iterator.", e); } + + } - + @Override public BibEntry next() { + System.out.println(" inside next -------------------------------"); + BibEntry entry = new Publication(); while (lineIterator.hasNext()) { String line = lineIterator.next(); - System.out.println(line + " 2 -------------------------------"); + if(line.contains("@")) { + entry.setArticleType(line.substring(line.indexOf('@')+1, line.indexOf('{'))); + System.out.println("type =========================== " +entry.getArticleType()); + } } return null; } @@ -67,7 +59,7 @@ public boolean hasNext() { @Override public void close() { // TODO Auto-generated method stub - + } } From f3cee647c5bae327963bbd75b2d07db331af0196 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 11 Oct 2024 16:32:53 -0700 Subject: [PATCH 13/49] [CITE-163] Parsed data for Bib Iterator --- .../parse/iterators/BibFileIterator.java | 136 ++++++++++++++++-- 1 file changed, 127 insertions(+), 9 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 1720226..d304067 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -2,13 +2,21 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; +import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; +import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; @@ -19,36 +27,146 @@ public class BibFileIterator implements BibEntryIterator { private String filePath; private Iterator lineIterator; private String currentLine = null; + private Map typeMap; public BibFileIterator(String filePath) { - System.out.println("------------------ inside bib file iterator"); this.filePath = filePath; init(); } private void init() { try { - System.out.println(filePath + " filepath ======================"); lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); } catch (IOException e) { logger.error("Could not create line iterator.", e); } - - + typeMap = new HashMap(); + typeMap.put("journal-article", Publication.ARTICLE); + typeMap.put("book", Publication.BOOK); + typeMap.put("book-chapter", Publication.BOOK_CHAPTER); + typeMap.put("monograph", Publication.BOOK); +// typeMap.put("journal-issue", Publication.JOURNAL_ISSUE); +// typeMap.put("reference-entry", Publication.REFERNCE_ENTRY); +// typeMap.put("posted-content", Publication.POSTED_CONTENT); +// typeMap.put("component", Publication.COMPONENT); +// typeMap.put("edited-book", Publication.EDITED_BOOK); + typeMap.put("proceedings-article", Publication.PROCEEDINGS_PAPER); +// typeMap.put("dissertation", Publication.DISSERTATION); + typeMap.put("book-section", Publication.BOOK_CHAPTER); +// typeMap.put("report-component", Publication.REPORT_COMPONENT); +// typeMap.put("report", Publication.REPORT); +// typeMap.put("peer-review", Publication.PEER_REVIEW); +// typeMap.put("book-track", Publication.BOOK_TRACK); +// typeMap.put("book-part", Publication.BOOK_PART); +// typeMap.put("other", Publication.OTHER); +// typeMap.put("journal-volume", Publication.JORUNAL_VOLUME); +// typeMap.put("book-set", Publication.BOOK_SET); +// typeMap.put("journal", Publication.JOURNAL); +// typeMap.put("proceedings-series", Publication.PROCEEDINGS_SERIES); +// typeMap.put("report-series", Publication.REPORT_SERIES); +// typeMap.put("proceedings", Publication.PROCEEDINGS); +// typeMap.put("database", Publication.DATABASE); +// typeMap.put("standard", Publication.STANDARD); +// typeMap.put("reference-book", Publication.REFERENCE_BOOK); +// typeMap.put("grant", Publication.GRANT); +// typeMap.put("dataset", Publication.DATASET); +// typeMap.put("book-series", Publication.BOOK_SERIES); } @Override public BibEntry next() { - System.out.println(" inside next -------------------------------"); BibEntry entry = new Publication(); + Map fields = new HashMap<>(); while (lineIterator.hasNext()) { - String line = lineIterator.next(); + String line = lineIterator.next().trim(); if(line.contains("@")) { - entry.setArticleType(line.substring(line.indexOf('@')+1, line.indexOf('{'))); - System.out.println("type =========================== " +entry.getArticleType()); + entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); + } else if (line.equals("}")) { + entry.setJournalMeta(parseJournalMeta(fields)); + entry.setArticleMeta(parseArticleMeta(fields)); + System.out.println("====================== entry - " + entry.toString()); + fields.clear(); + break; + } else if (line.contains("=")) { + String[] parts = line.split("=", 2); + if (parts.length == 2) { + String key = parts[0].trim(); + String value = parts[1].trim().replaceAll("[{},]", ""); // Remove curly braces and commas + fields.put(key, value); + } } } - return null; + return entry; + } + + private ContainerMeta parseJournalMeta(Map fields) { + ContainerMeta meta = new ContainerMeta(); +// meta.setContainerTitle(fields.get("title")); + meta.setPublisherName(fields.get("publisher")); + meta.setPublisherLocation(fields.get("place")); + List issnList = new ArrayList(); + String issnListString = fields.get("issn"); + for(String issnString : issnListString.split("and")) { + Issn issn = new Issn(); + issn.setIssn(issnString.trim()); + issnList.add(issn); + } + meta.setIssns(issnList); + return meta; + } + + private ArticleMeta parseArticleMeta(Map fields) { + ArticleMeta meta = new ArticleMeta(); + meta.setArticleTitle(fields.get("title")); +// meta.setShortTitle(fields.get("shortTitle")); + List contributors = new ArrayList<>(); + // List of authors + if(fields.get("author") != null) { +// contributors.addAll(mapPersonToContributor(item.getAuthor(), ContributionType.AUTHOR)); + System.out.println(fields.get("author")+ "========================== authors"); + } + // List of editors +// if(item.getEditor() != null) { +// contributors.addAll(mapPersonToContributor(item.getEditor(), ContributionType.EDITOR)); +// } +// // List of translators +// if(item.getTranslator() != null) { +// contributors.addAll(mapPersonToContributor(item.getTranslator(), ContributionType.TRANSLATOR)); +// } + // List of chair +// if(item.getChair() != null) { +// contributors.addAll(mapPersonToContributor(Arrays.asList(item.getChair()), ContributionType.CHAIR)); +// } +// meta.setContributors(contributors); + +// meta.setAuthorNotesCorrespondence(null); +// ArticlePublicationDate publicationDate = new ArticlePublicationDate(); +// List dateParts = item.getPublished().getIndexedDateParts(); +// if(dateParts != null) { +// publicationDate.setPublicationDate(dateParts.get(2).toString()); +// publicationDate.setPublicationMonth(dateParts.get(1).toString()); +// publicationDate.setPublicationYear(dateParts.get(0).toString()); +// } +// meta.setPublicationDate(publicationDate); + meta.setVolume(fields.get("volume")); + meta.setIssue(fields.get("issue")); + meta.setPartNumber(fields.get("partNumber")); + meta.setFirstPage(fields.get("firstPage")); + meta.setSelfUri(fields.get("uri")); + meta.setArticleAbstract(fields.get("abstract")); + meta.setLanguage(fields.get("language")); +// ReviewInfo review = new ReviewInfo(); +// if (item.getReview() != null) { +// review.setFullDescription(item.getReview().getCompetingInterestStatement()); +// } +// meta.setReviewInfo(review); + meta.setDocumentType(fields.get("type")); +// if(item.getReference() != null) { +// meta.setReferences(mapReferences(item.getReference())); +// } +// meta.setReferenceCount(item.getReferenceCount().toString()); + + return meta; } @Override From 46fc54ac4052964d539b893c14838bea08b2f601 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 15 Oct 2024 12:12:06 -0700 Subject: [PATCH 14/49] [CITE-163] Added code to parse author list --- .../parse/iterators/BibFileIterator.java | 158 ++++++++++-------- 1 file changed, 91 insertions(+), 67 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index d304067..cc31129 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -15,6 +15,7 @@ import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ContributionType; import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; @@ -45,32 +46,32 @@ private void init() { typeMap.put("book", Publication.BOOK); typeMap.put("book-chapter", Publication.BOOK_CHAPTER); typeMap.put("monograph", Publication.BOOK); -// typeMap.put("journal-issue", Publication.JOURNAL_ISSUE); -// typeMap.put("reference-entry", Publication.REFERNCE_ENTRY); -// typeMap.put("posted-content", Publication.POSTED_CONTENT); -// typeMap.put("component", Publication.COMPONENT); -// typeMap.put("edited-book", Publication.EDITED_BOOK); + // typeMap.put("journal-issue", Publication.JOURNAL_ISSUE); + // typeMap.put("reference-entry", Publication.REFERNCE_ENTRY); + // typeMap.put("posted-content", Publication.POSTED_CONTENT); + // typeMap.put("component", Publication.COMPONENT); + // typeMap.put("edited-book", Publication.EDITED_BOOK); typeMap.put("proceedings-article", Publication.PROCEEDINGS_PAPER); -// typeMap.put("dissertation", Publication.DISSERTATION); + // typeMap.put("dissertation", Publication.DISSERTATION); typeMap.put("book-section", Publication.BOOK_CHAPTER); -// typeMap.put("report-component", Publication.REPORT_COMPONENT); -// typeMap.put("report", Publication.REPORT); -// typeMap.put("peer-review", Publication.PEER_REVIEW); -// typeMap.put("book-track", Publication.BOOK_TRACK); -// typeMap.put("book-part", Publication.BOOK_PART); -// typeMap.put("other", Publication.OTHER); -// typeMap.put("journal-volume", Publication.JORUNAL_VOLUME); -// typeMap.put("book-set", Publication.BOOK_SET); -// typeMap.put("journal", Publication.JOURNAL); -// typeMap.put("proceedings-series", Publication.PROCEEDINGS_SERIES); -// typeMap.put("report-series", Publication.REPORT_SERIES); -// typeMap.put("proceedings", Publication.PROCEEDINGS); -// typeMap.put("database", Publication.DATABASE); -// typeMap.put("standard", Publication.STANDARD); -// typeMap.put("reference-book", Publication.REFERENCE_BOOK); -// typeMap.put("grant", Publication.GRANT); -// typeMap.put("dataset", Publication.DATASET); -// typeMap.put("book-series", Publication.BOOK_SERIES); + // typeMap.put("report-component", Publication.REPORT_COMPONENT); + // typeMap.put("report", Publication.REPORT); + // typeMap.put("peer-review", Publication.PEER_REVIEW); + // typeMap.put("book-track", Publication.BOOK_TRACK); + // typeMap.put("book-part", Publication.BOOK_PART); + // typeMap.put("other", Publication.OTHER); + // typeMap.put("journal-volume", Publication.JORUNAL_VOLUME); + // typeMap.put("book-set", Publication.BOOK_SET); + // typeMap.put("journal", Publication.JOURNAL); + // typeMap.put("proceedings-series", Publication.PROCEEDINGS_SERIES); + // typeMap.put("report-series", Publication.REPORT_SERIES); + // typeMap.put("proceedings", Publication.PROCEEDINGS); + // typeMap.put("database", Publication.DATABASE); + // typeMap.put("standard", Publication.STANDARD); + // typeMap.put("reference-book", Publication.REFERENCE_BOOK); + // typeMap.put("grant", Publication.GRANT); + // typeMap.put("dataset", Publication.DATASET); + // typeMap.put("book-series", Publication.BOOK_SERIES); } @Override @@ -91,63 +92,60 @@ public BibEntry next() { String[] parts = line.split("=", 2); if (parts.length == 2) { String key = parts[0].trim(); - String value = parts[1].trim().replaceAll("[{},]", ""); // Remove curly braces and commas + String value = parts[1].trim().replaceAll("\\{([^}]*)\\}", "$1"); // Remove curly braces and commas fields.put(key, value); } } } return entry; } - + private ContainerMeta parseJournalMeta(Map fields) { ContainerMeta meta = new ContainerMeta(); -// meta.setContainerTitle(fields.get("title")); + // meta.setContainerTitle(fields.get("title")); meta.setPublisherName(fields.get("publisher")); meta.setPublisherLocation(fields.get("place")); List issnList = new ArrayList(); - String issnListString = fields.get("issn"); - for(String issnString : issnListString.split("and")) { - Issn issn = new Issn(); - issn.setIssn(issnString.trim()); - issnList.add(issn); + if(fields.get("issn") != null) { + String issnListString = fields.get("issn"); + for(String issnString : issnListString.split("and")) { + Issn issn = new Issn(); + issn.setIssn(issnString.trim()); + issnList.add(issn); + } } meta.setIssns(issnList); return meta; } - + private ArticleMeta parseArticleMeta(Map fields) { ArticleMeta meta = new ArticleMeta(); meta.setArticleTitle(fields.get("title")); -// meta.setShortTitle(fields.get("shortTitle")); + // meta.setShortTitle(fields.get("shortTitle")); List contributors = new ArrayList<>(); // List of authors if(fields.get("author") != null) { -// contributors.addAll(mapPersonToContributor(item.getAuthor(), ContributionType.AUTHOR)); - System.out.println(fields.get("author")+ "========================== authors"); + contributors.addAll(mapPersonToContributor(fields.get("author"), ContributionType.AUTHOR)); + // System.out.println(fields.get("author")+ "========================== authors"); } // List of editors -// if(item.getEditor() != null) { -// contributors.addAll(mapPersonToContributor(item.getEditor(), ContributionType.EDITOR)); -// } -// // List of translators -// if(item.getTranslator() != null) { -// contributors.addAll(mapPersonToContributor(item.getTranslator(), ContributionType.TRANSLATOR)); -// } - // List of chair -// if(item.getChair() != null) { -// contributors.addAll(mapPersonToContributor(Arrays.asList(item.getChair()), ContributionType.CHAIR)); -// } -// meta.setContributors(contributors); - -// meta.setAuthorNotesCorrespondence(null); -// ArticlePublicationDate publicationDate = new ArticlePublicationDate(); -// List dateParts = item.getPublished().getIndexedDateParts(); -// if(dateParts != null) { -// publicationDate.setPublicationDate(dateParts.get(2).toString()); -// publicationDate.setPublicationMonth(dateParts.get(1).toString()); -// publicationDate.setPublicationYear(dateParts.get(0).toString()); -// } -// meta.setPublicationDate(publicationDate); + // if(item.getEditor() != null) { + // contributors.addAll(mapPersonToContributor(item.getEditor(), ContributionType.EDITOR)); + // } + // // List of translators + // if(item.getTranslator() != null) { + // contributors.addAll(mapPersonToContributor(item.getTranslator(), ContributionType.TRANSLATOR)); + // } + + // meta.setAuthorNotesCorrespondence(null); + // ArticlePublicationDate publicationDate = new ArticlePublicationDate(); + // List dateParts = item.getPublished().getIndexedDateParts(); + // if(dateParts != null) { + // publicationDate.setPublicationDate(dateParts.get(2).toString()); + // publicationDate.setPublicationMonth(dateParts.get(1).toString()); + // publicationDate.setPublicationYear(dateParts.get(0).toString()); + // } + // meta.setPublicationDate(publicationDate); meta.setVolume(fields.get("volume")); meta.setIssue(fields.get("issue")); meta.setPartNumber(fields.get("partNumber")); @@ -155,19 +153,45 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setSelfUri(fields.get("uri")); meta.setArticleAbstract(fields.get("abstract")); meta.setLanguage(fields.get("language")); -// ReviewInfo review = new ReviewInfo(); -// if (item.getReview() != null) { -// review.setFullDescription(item.getReview().getCompetingInterestStatement()); -// } -// meta.setReviewInfo(review); + // ReviewInfo review = new ReviewInfo(); + // if (item.getReview() != null) { + // review.setFullDescription(item.getReview().getCompetingInterestStatement()); + // } + // meta.setReviewInfo(review); meta.setDocumentType(fields.get("type")); -// if(item.getReference() != null) { -// meta.setReferences(mapReferences(item.getReference())); -// } -// meta.setReferenceCount(item.getReferenceCount().toString()); + // if(item.getReference() != null) { + // meta.setReferences(mapReferences(item.getReference())); + // } + // meta.setReferenceCount(item.getReferenceCount().toString()); return meta; } + + private List mapPersonToContributor(String authorsString, String contributionType) { + String[] authorStringList = authorsString.split("and"); + List contributors = new ArrayList(); + for(String person: authorStringList) { + Contributor contributor = new Contributor(); + String[] parts = person.split(","); + contributor.setContributionType(contributionType); + contributor.setGivenName(parts[0].trim()); + contributor.setSurname(parts[1].trim()); +// contributor.setFullName(person.getName()); +// List affiliations = new ArrayList<>(); +// for(Institution institute: person.getAffiliation()) { +// Affiliation affiliation = new Affiliation(); +// affiliation.setName(institute.getName()); +// affiliations.add(affiliation); +// } +// contributor.setAffiliations(affiliations); +// ContributorId contributorID = new ContributorId(); +// contributorID.setId(person.getOrcid()); +// contributorID.setIdSystem("ORCID"); +// contributor.setIds(Arrays.asList(contributorID)); + contributors.add(contributor); + } + return contributors; + } @Override public boolean hasNext() { From 318f193d3cd529acce51d450356d9bb291c8cf3c Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Wed, 16 Oct 2024 13:10:00 -0700 Subject: [PATCH 15/49] [CITE-163] Parsed some more details of citation --- .../parse/iterators/BibFileIterator.java | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index cc31129..e2f2118 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -9,11 +9,13 @@ import java.util.Map; import org.apache.commons.io.FileUtils; +import org.javers.common.collections.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ContributionType; import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; @@ -92,7 +94,7 @@ public BibEntry next() { String[] parts = line.split("=", 2); if (parts.length == 2) { String key = parts[0].trim(); - String value = parts[1].trim().replaceAll("\\{([^}]*)\\}", "$1"); // Remove curly braces and commas + String value = parts[1].trim().replaceAll("[{},]", ""); // Remove curly braces and commas fields.put(key, value); } } @@ -102,9 +104,14 @@ public BibEntry next() { private ContainerMeta parseJournalMeta(Map fields) { ContainerMeta meta = new ContainerMeta(); +// journalIds // meta.setContainerTitle(fields.get("title")); + List journalAbbrev = new ArrayList<>(); + journalAbbrev.add(fields.get("journal")); + meta.setJournalAbbreviations(journalAbbrev); meta.setPublisherName(fields.get("publisher")); meta.setPublisherLocation(fields.get("place")); +// publisherAddress List issnList = new ArrayList(); if(fields.get("issn") != null) { String issnListString = fields.get("issn"); @@ -115,6 +122,8 @@ private ContainerMeta parseJournalMeta(Map fields) { } } meta.setIssns(issnList); + meta.setSeriesTitle(fields.get("series")); +// private String seriesSubTitle; return meta; } @@ -129,27 +138,24 @@ private ArticleMeta parseArticleMeta(Map fields) { // System.out.println(fields.get("author")+ "========================== authors"); } // List of editors - // if(item.getEditor() != null) { - // contributors.addAll(mapPersonToContributor(item.getEditor(), ContributionType.EDITOR)); - // } - // // List of translators - // if(item.getTranslator() != null) { - // contributors.addAll(mapPersonToContributor(item.getTranslator(), ContributionType.TRANSLATOR)); - // } - + if(fields.get("editor") != null) { + contributors.addAll(mapPersonToContributor(fields.get("editor"), ContributionType.EDITOR)); + } + meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); - // ArticlePublicationDate publicationDate = new ArticlePublicationDate(); - // List dateParts = item.getPublished().getIndexedDateParts(); + ArticlePublicationDate publicationDate = new ArticlePublicationDate(); // if(dateParts != null) { // publicationDate.setPublicationDate(dateParts.get(2).toString()); // publicationDate.setPublicationMonth(dateParts.get(1).toString()); - // publicationDate.setPublicationYear(dateParts.get(0).toString()); + publicationDate.setPublicationYear(fields.get("year").trim()); // } - // meta.setPublicationDate(publicationDate); + meta.setPublicationDate(publicationDate); meta.setVolume(fields.get("volume")); - meta.setIssue(fields.get("issue")); - meta.setPartNumber(fields.get("partNumber")); - meta.setFirstPage(fields.get("firstPage")); + meta.setIssue(fields.get("issue")); // no eg + meta.setIssueId(fields.get("issueId")); //no eg + meta.setPartNumber(fields.get("partNumber")); // no eg + meta.setFirstPage(fields.get("pages").split("--")[0].trim()); + meta.setLastPage(fields.get("pages").split("--")[1].trim()); meta.setSelfUri(fields.get("uri")); meta.setArticleAbstract(fields.get("abstract")); meta.setLanguage(fields.get("language")); @@ -167,16 +173,16 @@ private ArticleMeta parseArticleMeta(Map fields) { return meta; } - private List mapPersonToContributor(String authorsString, String contributionType) { - String[] authorStringList = authorsString.split("and"); + private List mapPersonToContributor(String contributorsString, String contributionType) { + String[] contributorStringList = contributorsString.split("and"); List contributors = new ArrayList(); - for(String person: authorStringList) { + for(String personStr: contributorStringList) { Contributor contributor = new Contributor(); - String[] parts = person.split(","); + String[] parts = personStr.split(" "); contributor.setContributionType(contributionType); - contributor.setGivenName(parts[0].trim()); - contributor.setSurname(parts[1].trim()); -// contributor.setFullName(person.getName()); + contributor.setGivenName(parts[1].trim()); + contributor.setSurname(parts[0].trim()); + contributor.setFullName(parts[1].trim()+" "+parts[0].trim()); // List affiliations = new ArrayList<>(); // for(Institution institute: person.getAffiliation()) { // Affiliation affiliation = new Affiliation(); From d281d39232820b607d290daac536c5cd8feaea97 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 17 Oct 2024 12:09:18 -0700 Subject: [PATCH 16/49] [CITE-163] Created item from bib file text --- .../service/parse/impl/BibFileHandler.java | 2 +- .../parse/iterators/BibFileIterator.java | 47 +++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index 0d75296..2e42755 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -25,7 +25,7 @@ public boolean canHandle(String path) throws HandlerTestException { public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) throws IteratorCreationException { System.out.println("inside bib file handler ................."); - return new BibFileIterator(path); + return new BibFileIterator(path, info.getGroupId()); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index e2f2118..1092f42 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -9,9 +9,11 @@ import java.util.Map; import org.apache.commons.io.FileUtils; -import org.javers.common.collections.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.social.zotero.api.Data; +import org.springframework.social.zotero.api.Item; +import org.springframework.social.zotero.api.Library; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; @@ -28,12 +30,14 @@ public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); private String filePath; + private String groupId; private Iterator lineIterator; private String currentLine = null; private Map typeMap; - public BibFileIterator(String filePath) { + public BibFileIterator(String filePath, String groupId) { this.filePath = filePath; + this.groupId = groupId; init(); } @@ -80,14 +84,21 @@ private void init() { public BibEntry next() { BibEntry entry = new Publication(); Map fields = new HashMap<>(); + Item item = new Item(); + Library lib = new Library(); + lib.setId(new Long(groupId)); + item.setLibrary(lib); + Data itemData = new Data(); while (lineIterator.hasNext()) { String line = lineIterator.next().trim(); if(line.contains("@")) { - entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); +// entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); + itemData.setItemType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); } else if (line.equals("}")) { - entry.setJournalMeta(parseJournalMeta(fields)); - entry.setArticleMeta(parseArticleMeta(fields)); - System.out.println("====================== entry - " + entry.toString()); +// entry.setJournalMeta(parseJournalMeta(fields)); +// entry.setArticleMeta(parseArticleMeta(fields)); + itemData = parseItemData(itemData, fields); +// System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; } else if (line.contains("=")) { @@ -99,8 +110,29 @@ public BibEntry next() { } } } + item.setData(itemData); return entry; } + + private Data parseItemData(Data data, Map fields) { +// linkMode; accessDate contentType charset filename md5 mtime dateAdded dateModified +// creators publicationTitle issue date seriesTitle seriesText archive +// archiveLocation libraryCatalog callNumber rights extra tags collections + data.setTitle(fields.get("title")); + data.setUrl(fields.get("url")); + data.setNote(fields.get("note")); + data.setAbstractNote(fields.get("abstract")); + data.setVolume(fields.get("volume")); + data.setPages(fields.get("pages")); + data.setSeries(fields.get("series")); + data.setJournalAbbreviation(fields.get("journal")); + data.setLanguage(fields.get("language")); + data.setDoi(fields.get("doi")); + data.setIssn(fields.get("issn")); + data.setShortTitle(fields.get("shorttitle")); + + return data; + } private ContainerMeta parseJournalMeta(Map fields) { ContainerMeta meta = new ContainerMeta(); @@ -151,9 +183,6 @@ private ArticleMeta parseArticleMeta(Map fields) { // } meta.setPublicationDate(publicationDate); meta.setVolume(fields.get("volume")); - meta.setIssue(fields.get("issue")); // no eg - meta.setIssueId(fields.get("issueId")); //no eg - meta.setPartNumber(fields.get("partNumber")); // no eg meta.setFirstPage(fields.get("pages").split("--")[0].trim()); meta.setLastPage(fields.get("pages").split("--")[1].trim()); meta.setSelfUri(fields.get("uri")); From 91925ece1dcf04a92d3de0a7d08b0f6678361d61 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 18 Oct 2024 16:40:25 -0700 Subject: [PATCH 17/49] [CITE-163] Parsed Citation to BibEntry in BibFileIterator --- .../importer/core/model/impl/ArticleMeta.java | 7 ++ .../parse/iterators/BibFileIterator.java | 77 +++++++++++-------- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index 1b403f5..ec3f27d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -7,6 +7,7 @@ public class ArticleMeta { private List articleIds; private String articleTitle; + private String articleShortTitle; private List categoryGroups = new ArrayList(); private List contributors; private String authorNotesCorrespondence; @@ -56,6 +57,12 @@ public String getArticleTitle() { public void setArticleTitle(String articleTitle) { this.articleTitle = articleTitle; } + public String getArticleShortTitle() { + return articleShortTitle; + } + public void setArticleShortTitle(String articleShortTitle) { + this.articleShortTitle = articleShortTitle; + } public List getCategories() { return categoryGroups; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 1092f42..a8c7281 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -11,10 +11,12 @@ import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.social.zotero.api.Data; import org.springframework.social.zotero.api.Item; import org.springframework.social.zotero.api.Library; +import edu.asu.diging.citesphere.factory.impl.CitationFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; @@ -24,10 +26,14 @@ import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.model.bib.ICitation; public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Autowired + private CitationFactory citationFactory; private String filePath; private String groupId; @@ -86,18 +92,23 @@ public BibEntry next() { Map fields = new HashMap<>(); Item item = new Item(); Library lib = new Library(); - lib.setId(new Long(groupId)); + lib.setId(Long.parseLong(groupId)); item.setLibrary(lib); Data itemData = new Data(); while (lineIterator.hasNext()) { String line = lineIterator.next().trim(); if(line.contains("@")) { -// entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); + entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); itemData.setItemType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); } else if (line.equals("}")) { // entry.setJournalMeta(parseJournalMeta(fields)); // entry.setArticleMeta(parseArticleMeta(fields)); itemData = parseItemData(itemData, fields); + item.setData(itemData); + ICitation citation = citationFactory.createCitation(item); + // convert citation to bibentry + entry.setJournalMeta(parseJournalMeta(citation, fields)); + entry.setArticleMeta(parseArticleMeta(citation, fields)); // System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; @@ -110,15 +121,16 @@ public BibEntry next() { } } } - item.setData(itemData); + return entry; } private Data parseItemData(Data data, Map fields) { // linkMode; accessDate contentType charset filename md5 mtime dateAdded dateModified -// creators publicationTitle issue date seriesTitle seriesText archive +// creators publicationTitle seriesTitle seriesText archive // archiveLocation libraryCatalog callNumber rights extra tags collections data.setTitle(fields.get("title")); + data.setPublicationTitle(fields.get("publisher")); data.setUrl(fields.get("url")); data.setNote(fields.get("note")); data.setAbstractNote(fields.get("abstract")); @@ -130,64 +142,67 @@ private Data parseItemData(Data data, Map fields) { data.setDoi(fields.get("doi")); data.setIssn(fields.get("issn")); data.setShortTitle(fields.get("shorttitle")); + data.setDate(fields.get("year")+"-01-01T00:00Z"); + data.setIssue(fields.get("number")); return data; } - private ContainerMeta parseJournalMeta(Map fields) { + private ContainerMeta parseJournalMeta(ICitation citation, Map fields) { ContainerMeta meta = new ContainerMeta(); // journalIds // meta.setContainerTitle(fields.get("title")); List journalAbbrev = new ArrayList<>(); - journalAbbrev.add(fields.get("journal")); + journalAbbrev.add(citation.getJournalAbbreviation()); meta.setJournalAbbreviations(journalAbbrev); - meta.setPublisherName(fields.get("publisher")); + meta.setPublisherName(citation.getPublicationTitle()); meta.setPublisherLocation(fields.get("place")); // publisherAddress List issnList = new ArrayList(); if(fields.get("issn") != null) { - String issnListString = fields.get("issn"); - for(String issnString : issnListString.split("and")) { + for(String issnString : fields.get("issn").split("and")) { Issn issn = new Issn(); issn.setIssn(issnString.trim()); issnList.add(issn); } } meta.setIssns(issnList); - meta.setSeriesTitle(fields.get("series")); + meta.setSeriesTitle(citation.getSeriesTitle()); // private String seriesSubTitle; return meta; } - private ArticleMeta parseArticleMeta(Map fields) { + private ArticleMeta parseArticleMeta(ICitation citation, Map fields) { ArticleMeta meta = new ArticleMeta(); - meta.setArticleTitle(fields.get("title")); - // meta.setShortTitle(fields.get("shortTitle")); + meta.setArticleTitle(citation.getTitle()); + meta.setArticleShortTitle(citation.getShortTitle()); +// categoryGroups List contributors = new ArrayList<>(); // List of authors - if(fields.get("author") != null) { - contributors.addAll(mapPersonToContributor(fields.get("author"), ContributionType.AUTHOR)); - // System.out.println(fields.get("author")+ "========================== authors"); - } +// if(fields.get("author") != null) { +// contributors.addAll(mapPersonToContributor(fields.get("author"), ContributionType.AUTHOR)); +// // System.out.println(fields.get("author")+ "========================== authors"); +// } // List of editors - if(fields.get("editor") != null) { - contributors.addAll(mapPersonToContributor(fields.get("editor"), ContributionType.EDITOR)); - } - meta.setContributors(contributors); +// if(fields.get("editor") != null) { +// contributors.addAll(mapPersonToContributor(fields.get("editor"), ContributionType.EDITOR)); +// } +// meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); ArticlePublicationDate publicationDate = new ArticlePublicationDate(); - // if(dateParts != null) { - // publicationDate.setPublicationDate(dateParts.get(2).toString()); - // publicationDate.setPublicationMonth(dateParts.get(1).toString()); - publicationDate.setPublicationYear(fields.get("year").trim()); +// if(citation != null) { +// publicationDate.setPublicationDate(dateParts.get(2).toString()); +// publicationDate.setPublicationMonth(dateParts.get(1).toString()); + publicationDate.setPublicationYear(citation.getDateFreetext()); // } meta.setPublicationDate(publicationDate); - meta.setVolume(fields.get("volume")); - meta.setFirstPage(fields.get("pages").split("--")[0].trim()); - meta.setLastPage(fields.get("pages").split("--")[1].trim()); - meta.setSelfUri(fields.get("uri")); - meta.setArticleAbstract(fields.get("abstract")); - meta.setLanguage(fields.get("language")); + meta.setVolume(citation.getVolume()); + meta.setIssue(citation.getIssue()); + meta.setFirstPage(citation.getPages().split("--")[0].trim()); + meta.setLastPage(citation.getPages().split("--")[1].trim()); + meta.setSelfUri(citation.getUrl()); + meta.setArticleAbstract(citation.getAbstractNote()); + meta.setLanguage(citation.getLanguage()); // ReviewInfo review = new ReviewInfo(); // if (item.getReview() != null) { // review.setFullDescription(item.getReview().getCompetingInterestStatement()); From 3d2802f7ac342646300368147b5d245c94996996 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 22 Oct 2024 12:04:56 -0700 Subject: [PATCH 18/49] [CITE-163] Mapped references and authors from citation to bibentry --- .../importer/core/model/impl/Affiliation.java | 20 +++ .../parse/iterators/BibFileIterator.java | 150 ++++++++++++------ 2 files changed, 118 insertions(+), 52 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java index 120a8f8..818474e 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Affiliation.java @@ -3,6 +3,8 @@ public class Affiliation { private String name; + private String uri; + private String localAuthorityId; public String getName() { return name; @@ -11,4 +13,22 @@ public String getName() { public void setName(String name) { this.name = name; } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getLocalAuthorityId() { + return localAuthorityId; + } + + public void setLocalAuthorityId(String localAuthorityId) { + this.localAuthorityId = localAuthorityId; + } + + } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index a8c7281..064efa6 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -7,6 +7,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -18,6 +19,7 @@ import edu.asu.diging.citesphere.factory.impl.CitationFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; +import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; @@ -25,13 +27,17 @@ import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; +import edu.asu.diging.citesphere.importer.core.model.impl.Reference; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; +import edu.asu.diging.citesphere.model.bib.IAffiliation; import edu.asu.diging.citesphere.model.bib.ICitation; +import edu.asu.diging.citesphere.model.bib.IPerson; +import edu.asu.diging.citesphere.model.bib.IReference; public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); - + @Autowired private CitationFactory citationFactory; @@ -101,15 +107,15 @@ public BibEntry next() { entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); itemData.setItemType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); } else if (line.equals("}")) { -// entry.setJournalMeta(parseJournalMeta(fields)); -// entry.setArticleMeta(parseArticleMeta(fields)); + // entry.setJournalMeta(parseJournalMeta(fields)); + // entry.setArticleMeta(parseArticleMeta(fields)); itemData = parseItemData(itemData, fields); item.setData(itemData); ICitation citation = citationFactory.createCitation(item); // convert citation to bibentry entry.setJournalMeta(parseJournalMeta(citation, fields)); - entry.setArticleMeta(parseArticleMeta(citation, fields)); -// System.out.println("====================== entry - " + entry.toString()); + entry.setArticleMeta(parseArticleMeta(citation, fields)); + // System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; } else if (line.contains("=")) { @@ -124,11 +130,11 @@ public BibEntry next() { return entry; } - + private Data parseItemData(Data data, Map fields) { -// linkMode; accessDate contentType charset filename md5 mtime dateAdded dateModified -// creators publicationTitle seriesTitle seriesText archive -// archiveLocation libraryCatalog callNumber rights extra tags collections + // linkMode; accessDate contentType charset filename md5 mtime dateAdded dateModified + // creators publicationTitle seriesTitle seriesText archive + // archiveLocation libraryCatalog callNumber rights extra tags collections data.setTitle(fields.get("title")); data.setPublicationTitle(fields.get("publisher")); data.setUrl(fields.get("url")); @@ -144,20 +150,20 @@ private Data parseItemData(Data data, Map fields) { data.setShortTitle(fields.get("shorttitle")); data.setDate(fields.get("year")+"-01-01T00:00Z"); data.setIssue(fields.get("number")); - + return data; } private ContainerMeta parseJournalMeta(ICitation citation, Map fields) { ContainerMeta meta = new ContainerMeta(); -// journalIds + // journalIds // meta.setContainerTitle(fields.get("title")); List journalAbbrev = new ArrayList<>(); journalAbbrev.add(citation.getJournalAbbreviation()); meta.setJournalAbbreviations(journalAbbrev); meta.setPublisherName(citation.getPublicationTitle()); meta.setPublisherLocation(fields.get("place")); -// publisherAddress + // publisherAddress List issnList = new ArrayList(); if(fields.get("issn") != null) { for(String issnString : fields.get("issn").split("and")) { @@ -168,7 +174,7 @@ private ContainerMeta parseJournalMeta(ICitation citation, Map f } meta.setIssns(issnList); meta.setSeriesTitle(citation.getSeriesTitle()); -// private String seriesSubTitle; + // private String seriesSubTitle; return meta; } @@ -176,26 +182,26 @@ private ArticleMeta parseArticleMeta(ICitation citation, Map fie ArticleMeta meta = new ArticleMeta(); meta.setArticleTitle(citation.getTitle()); meta.setArticleShortTitle(citation.getShortTitle()); -// categoryGroups + // categoryGroups List contributors = new ArrayList<>(); // List of authors -// if(fields.get("author") != null) { -// contributors.addAll(mapPersonToContributor(fields.get("author"), ContributionType.AUTHOR)); -// // System.out.println(fields.get("author")+ "========================== authors"); -// } + if(citation.getAuthors() != null) { + contributors.addAll(mapPersonToContributor(citation.getAuthors(), ContributionType.AUTHOR)); + // System.out.println(fields.get("author")+ "========================== authors"); + } // List of editors -// if(fields.get("editor") != null) { -// contributors.addAll(mapPersonToContributor(fields.get("editor"), ContributionType.EDITOR)); -// } -// meta.setContributors(contributors); + // if(fields.get("editor") != null) { + // contributors.addAll(mapPersonToContributor(fields.get("editor"), ContributionType.EDITOR)); + // } + // meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); - ArticlePublicationDate publicationDate = new ArticlePublicationDate(); -// if(citation != null) { -// publicationDate.setPublicationDate(dateParts.get(2).toString()); -// publicationDate.setPublicationMonth(dateParts.get(1).toString()); - publicationDate.setPublicationYear(citation.getDateFreetext()); + ArticlePublicationDate publicationDate = new ArticlePublicationDate(); + // if(citation != null) { + // publicationDate.setPublicationDate(dateParts.get(2).toString()); + // publicationDate.setPublicationMonth(dateParts.get(1).toString()); + publicationDate.setPublicationYear(citation.getDateFreetext()); // } - meta.setPublicationDate(publicationDate); + meta.setPublicationDate(publicationDate); meta.setVolume(citation.getVolume()); meta.setIssue(citation.getIssue()); meta.setFirstPage(citation.getPages().split("--")[0].trim()); @@ -208,41 +214,81 @@ private ArticleMeta parseArticleMeta(ICitation citation, Map fie // review.setFullDescription(item.getReview().getCompetingInterestStatement()); // } // meta.setReviewInfo(review); - meta.setDocumentType(fields.get("type")); - // if(item.getReference() != null) { - // meta.setReferences(mapReferences(item.getReference())); - // } - // meta.setReferenceCount(item.getReferenceCount().toString()); + // meta.setDocumentType(fields.get("type")); // document type in note + // conferenceInfo in note - map to conferenceTitle, conferenceDate, conferenceLocation, conferenceSponsor, conferenceHost + // keywords in note + // reprintAddress in note + // additionalData in note + // funding in note - map to fundingInfo, fundingText + + if(citation.getReferences() != null) { + meta.setReferences(mapReferences(citation.getReferences())); + } + meta.setReferenceCount(meta.getReferences().size()+""); + + // retrievalDate in note return meta; } - - private List mapPersonToContributor(String contributorsString, String contributionType) { - String[] contributorStringList = contributorsString.split("and"); + + private List mapPersonToContributor(Set citationContributors, String contributionType) { + // String[] contributorStringList = contributorsString.split("and"); List contributors = new ArrayList(); - for(String personStr: contributorStringList) { + for(IPerson person: citationContributors) { Contributor contributor = new Contributor(); - String[] parts = personStr.split(" "); contributor.setContributionType(contributionType); - contributor.setGivenName(parts[1].trim()); - contributor.setSurname(parts[0].trim()); - contributor.setFullName(parts[1].trim()+" "+parts[0].trim()); -// List affiliations = new ArrayList<>(); -// for(Institution institute: person.getAffiliation()) { -// Affiliation affiliation = new Affiliation(); -// affiliation.setName(institute.getName()); -// affiliations.add(affiliation); -// } -// contributor.setAffiliations(affiliations); -// ContributorId contributorID = new ContributorId(); -// contributorID.setId(person.getOrcid()); -// contributorID.setIdSystem("ORCID"); -// contributor.setIds(Arrays.asList(contributorID)); + contributor.setGivenName(person.getFirstName()); + contributor.setSurname(person.getLastName()); + contributor.setFullName(person.getName()); + + List affiliations = new ArrayList<>(); + for(IAffiliation institute: person.getAffiliations()) { + Affiliation affiliation = new Affiliation(); + affiliation.setName(institute.getName()); + affiliation.setUri(institute.getUri()); + affiliation.setLocalAuthorityId(institute.getLocalAuthorityId()); + affiliations.add(affiliation); + } + contributor.setAffiliations(affiliations); + // ContributorId contributorID = new ContributorId(); + // contributorID.setId(person.getOrcid()); + // contributorID.setIdSystem("ORCID"); + // contributor.setIds(Arrays.asList(contributorID)); contributors.add(contributor); } return contributors; } + private List mapReferences(Set citationReferences) { + List references = new ArrayList(); + for(IReference citationRef: citationReferences) { + references.add(mapSingleReference(citationRef)); + } + + return references; + } + + private Reference mapSingleReference(IReference citationRef) { + Reference ref = new Reference(); + ref.setAuthorString(citationRef.getAuthorString()); + ref.setTitle(citationRef.getTitle()); + ref.setYear(citationRef.getYear()); + ref.setIdentifier(citationRef.getIdentifier()); + ref.setIdentifierType(citationRef.getIdentifierType()); + ref.setFirstPage(citationRef.getFirstPage()); + ref.setEndPage(citationRef.getEndPage()); + ref.setVolume(citationRef.getVolume()); + ref.setSource(citationRef.getSource()); + ref.setReferenceId(citationRef.getReferenceId()); + ref.setReferenceLabel(citationRef.getReferenceLabel()); + ref.setPublicationType(citationRef.getPublicationType()); + ref.setCitationId(citationRef.getCitationId()); + ref.setReferenceString(citationRef.getReferenceString()); + ref.setReferenceStringRaw(citationRef.getReferenceStringRaw()); + + return ref; + } + @Override public boolean hasNext() { return lineIterator.hasNext(); From 3717bf49d55845f5595ae6ce6e8bbc85e33b5eaf Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Wed, 23 Oct 2024 13:10:37 -0700 Subject: [PATCH 19/49] [CITE-163] Solved some errors that bib Iterator --- citesphere-importer/pom.xml | 2 +- .../importer/core/model/impl/Contributor.java | 10 ++++++ .../parse/iterators/BibFileIterator.java | 34 +++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/citesphere-importer/pom.xml b/citesphere-importer/pom.xml index 5c8d936..bb6727d 100644 --- a/citesphere-importer/pom.xml +++ b/citesphere-importer/pom.xml @@ -16,7 +16,7 @@ 3.0.11.RELEASE 2.2.6.RELEASE 0.5 - 1.12 + 1.22 2.4.0.RELEASE $2a$04$oQo44vqcDIFRoYKiAXoNheurzkwX9dcNmowvTX/hsWuBMwijqn44i diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java index 623720c..1af7730 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/Contributor.java @@ -13,6 +13,7 @@ public class Contributor { private String fullSurname; private String fullName; private String emailAddress; + private String uri; private List affiliations = new ArrayList(); private List ids; @@ -64,6 +65,15 @@ public String getEmailAddress() { public void setEmailAddress(String emailAddress) { this.emailAddress = emailAddress; } + public String getUri() { + return uri; + } + public void setUri(String uri) { + this.uri = uri; + } + public void setFullStandardizeName(String fullStandardizeName) { + this.fullStandardizeName = fullStandardizeName; + } public List getAffiliations() { return affiliations; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 064efa6..ddcd9b2 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -17,7 +17,7 @@ import org.springframework.social.zotero.api.Item; import org.springframework.social.zotero.api.Library; -import edu.asu.diging.citesphere.factory.impl.CitationFactory; +import edu.asu.diging.citesphere.factory.ICitationFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; @@ -39,7 +39,7 @@ public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); @Autowired - private CitationFactory citationFactory; + private ICitationFactory citationFactory; private String filePath; private String groupId; @@ -60,7 +60,7 @@ private void init() { logger.error("Could not create line iterator.", e); } typeMap = new HashMap(); - typeMap.put("journal-article", Publication.ARTICLE); + typeMap.put("article", Publication.ARTICLE); typeMap.put("book", Publication.BOOK); typeMap.put("book-chapter", Publication.BOOK_CHAPTER); typeMap.put("monograph", Publication.BOOK); @@ -103,19 +103,23 @@ public BibEntry next() { Data itemData = new Data(); while (lineIterator.hasNext()) { String line = lineIterator.next().trim(); - if(line.contains("@")) { - entry.setArticleType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); - itemData.setItemType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); +// System.out.println("============ line - "+ line); + if(!line.isBlank() && line.charAt(0)=='@') { + System.out.println("===================" + line.substring(1, line.indexOf('{'))); + entry.setArticleType(typeMap.get(line.substring(1, line.indexOf('{')))); +// itemData.setItemType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); } else if (line.equals("}")) { // entry.setJournalMeta(parseJournalMeta(fields)); // entry.setArticleMeta(parseArticleMeta(fields)); itemData = parseItemData(itemData, fields); item.setData(itemData); - ICitation citation = citationFactory.createCitation(item); + System.out.println("item ================= " +item); +// System.out.println("citation factory ===================== "+ citationFactory); + ICitation citation = citationFactory.createCitation(item, null); // convert citation to bibentry entry.setJournalMeta(parseJournalMeta(citation, fields)); entry.setArticleMeta(parseArticleMeta(citation, fields)); - // System.out.println("====================== entry - " + entry.toString()); + System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; } else if (line.contains("=")) { @@ -190,10 +194,10 @@ private ArticleMeta parseArticleMeta(ICitation citation, Map fie // System.out.println(fields.get("author")+ "========================== authors"); } // List of editors - // if(fields.get("editor") != null) { - // contributors.addAll(mapPersonToContributor(fields.get("editor"), ContributionType.EDITOR)); - // } - // meta.setContributors(contributors); + if(citation.getEditors() != null) { + contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); + } + meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); ArticlePublicationDate publicationDate = new ArticlePublicationDate(); // if(citation != null) { @@ -240,6 +244,7 @@ private List mapPersonToContributor(Set citationContributo contributor.setGivenName(person.getFirstName()); contributor.setSurname(person.getLastName()); contributor.setFullName(person.getName()); + contributor.setUri(person.getUri()); List affiliations = new ArrayList<>(); for(IAffiliation institute: person.getAffiliations()) { @@ -250,11 +255,6 @@ private List mapPersonToContributor(Set citationContributo affiliations.add(affiliation); } contributor.setAffiliations(affiliations); - // ContributorId contributorID = new ContributorId(); - // contributorID.setId(person.getOrcid()); - // contributorID.setIdSystem("ORCID"); - // contributor.setIds(Arrays.asList(contributorID)); - contributors.add(contributor); } return contributors; } From 2fb3a4f46a499a4bd391787804648124be98d0b0 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 24 Oct 2024 12:09:57 -0700 Subject: [PATCH 20/49] [CITE-163] Changed the logic to parse extra/note --- .../importer/core/model/impl/ArticleMeta.java | 7 ++ .../parse/iterators/BibFileIterator.java | 81 ++++++++----------- 2 files changed, 39 insertions(+), 49 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index ec3f27d..74bc8e7 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -26,6 +26,7 @@ public class ArticleMeta { private String copyrightYear; private String copyrightHolder; private String selfUri; + private String doi; private String articleAbstract; private String language; private ReviewInfo reviewInfo; @@ -171,6 +172,12 @@ public String getSelfUri() { public void setSelfUri(String selfUri) { this.selfUri = selfUri; } + public String getDoi() { + return doi; + } + public void setDoi(String doi) { + this.doi = doi; + } public String getArticleAbstract() { return articleAbstract; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index ddcd9b2..253548d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -12,18 +12,15 @@ import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.social.zotero.api.Data; import org.springframework.social.zotero.api.Item; import org.springframework.social.zotero.api.Library; -import edu.asu.diging.citesphere.factory.ICitationFactory; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; -import edu.asu.diging.citesphere.importer.core.model.impl.ContributionType; import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; @@ -33,13 +30,13 @@ import edu.asu.diging.citesphere.model.bib.ICitation; import edu.asu.diging.citesphere.model.bib.IPerson; import edu.asu.diging.citesphere.model.bib.IReference; +import edu.asu.diging.citesphere.model.bib.impl.Citation; public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired - private ICitationFactory citationFactory; + private ParseExtra parseExtra; private String filePath; private String groupId; @@ -50,6 +47,7 @@ public class BibFileIterator implements BibEntryIterator { public BibFileIterator(String filePath, String groupId) { this.filePath = filePath; this.groupId = groupId; + parseExtra = new ParseExtra(); init(); } @@ -111,14 +109,14 @@ public BibEntry next() { } else if (line.equals("}")) { // entry.setJournalMeta(parseJournalMeta(fields)); // entry.setArticleMeta(parseArticleMeta(fields)); - itemData = parseItemData(itemData, fields); +// itemData = parseItemData(itemData, fields); item.setData(itemData); System.out.println("item ================= " +item); // System.out.println("citation factory ===================== "+ citationFactory); - ICitation citation = citationFactory.createCitation(item, null); +// ICitation citation = citationFactory.createCitation(item, null); // convert citation to bibentry - entry.setJournalMeta(parseJournalMeta(citation, fields)); - entry.setArticleMeta(parseArticleMeta(citation, fields)); + entry.setJournalMeta(parseJournalMeta(fields)); + entry.setArticleMeta(parseArticleMeta(fields)); System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; @@ -135,37 +133,14 @@ public BibEntry next() { return entry; } - private Data parseItemData(Data data, Map fields) { - // linkMode; accessDate contentType charset filename md5 mtime dateAdded dateModified - // creators publicationTitle seriesTitle seriesText archive - // archiveLocation libraryCatalog callNumber rights extra tags collections - data.setTitle(fields.get("title")); - data.setPublicationTitle(fields.get("publisher")); - data.setUrl(fields.get("url")); - data.setNote(fields.get("note")); - data.setAbstractNote(fields.get("abstract")); - data.setVolume(fields.get("volume")); - data.setPages(fields.get("pages")); - data.setSeries(fields.get("series")); - data.setJournalAbbreviation(fields.get("journal")); - data.setLanguage(fields.get("language")); - data.setDoi(fields.get("doi")); - data.setIssn(fields.get("issn")); - data.setShortTitle(fields.get("shorttitle")); - data.setDate(fields.get("year")+"-01-01T00:00Z"); - data.setIssue(fields.get("number")); - - return data; - } - - private ContainerMeta parseJournalMeta(ICitation citation, Map fields) { + private ContainerMeta parseJournalMeta(Map fields) { ContainerMeta meta = new ContainerMeta(); // journalIds // meta.setContainerTitle(fields.get("title")); List journalAbbrev = new ArrayList<>(); - journalAbbrev.add(citation.getJournalAbbreviation()); + journalAbbrev.add(fields.get("journal")); meta.setJournalAbbreviations(journalAbbrev); - meta.setPublisherName(citation.getPublicationTitle()); + meta.setPublisherName(fields.get("publisher")); meta.setPublisherLocation(fields.get("place")); // publisherAddress List issnList = new ArrayList(); @@ -177,25 +152,32 @@ private ContainerMeta parseJournalMeta(ICitation citation, Map f } } meta.setIssns(issnList); - meta.setSeriesTitle(citation.getSeriesTitle()); + meta.setSeriesTitle(fields.get("series")); // private String seriesSubTitle; return meta; } - private ArticleMeta parseArticleMeta(ICitation citation, Map fields) { + private ArticleMeta parseArticleMeta(Map fields) { ArticleMeta meta = new ArticleMeta(); - meta.setArticleTitle(citation.getTitle()); - meta.setArticleShortTitle(citation.getShortTitle()); + ICitation citation = new Citation(); + Item item = new Item(); + Data data = new Data(); + data.setNote(fields.get("note")); + item.setData(data); + parseExtra.parseMetaDataNote(citation, item); + + meta.setArticleTitle(fields.get("title")); + meta.setArticleShortTitle(fields.get("shorttitle")); // categoryGroups List contributors = new ArrayList<>(); // List of authors if(citation.getAuthors() != null) { - contributors.addAll(mapPersonToContributor(citation.getAuthors(), ContributionType.AUTHOR)); +// contributors.addAll(mapPersonToContributor(citation.getAuthors(), ContributionType.AUTHOR)); // System.out.println(fields.get("author")+ "========================== authors"); } // List of editors if(citation.getEditors() != null) { - contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); +// contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); } meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); @@ -203,16 +185,17 @@ private ArticleMeta parseArticleMeta(ICitation citation, Map fie // if(citation != null) { // publicationDate.setPublicationDate(dateParts.get(2).toString()); // publicationDate.setPublicationMonth(dateParts.get(1).toString()); - publicationDate.setPublicationYear(citation.getDateFreetext()); + publicationDate.setPublicationYear(fields.get("year")); // } meta.setPublicationDate(publicationDate); - meta.setVolume(citation.getVolume()); - meta.setIssue(citation.getIssue()); - meta.setFirstPage(citation.getPages().split("--")[0].trim()); - meta.setLastPage(citation.getPages().split("--")[1].trim()); - meta.setSelfUri(citation.getUrl()); - meta.setArticleAbstract(citation.getAbstractNote()); - meta.setLanguage(citation.getLanguage()); + meta.setVolume(fields.get("volume")); + meta.setIssue(fields.get("number")); + meta.setFirstPage(fields.get("pages").split("--")[0].trim()); + meta.setLastPage(fields.get("pages").split("--")[1].trim()); + meta.setSelfUri(fields.get("url")); + meta.setDoi(fields.get("doi")); + meta.setArticleAbstract(fields.get("abstract")); + meta.setLanguage(fields.get("language")); // ReviewInfo review = new ReviewInfo(); // if (item.getReview() != null) { // review.setFullDescription(item.getReview().getCompetingInterestStatement()); From 1477e37605d44fa58d40ff97406e5c54ac721290 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 25 Oct 2024 16:07:37 -0700 Subject: [PATCH 21/49] [CITE-163] Cleaned code and resolved error related to parsing --- .../service/giles/GilesUploadService.java | 11 -- .../giles/impl/GilesUploadServiceImpl.java | 97 -------------- .../service/oauth/InternalTokenManager.java | 11 -- .../oauth/impl/InternalTokenManagerImpl.java | 121 ------------------ .../parse/iterators/BibFileIterator.java | 8 +- 5 files changed, 7 insertions(+), 241 deletions(-) delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java deleted file mode 100644 index ef0f484..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/GilesUploadService.java +++ /dev/null @@ -1,11 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.giles; - -import org.springframework.web.multipart.MultipartFile; - -import edu.asu.diging.citesphere.model.bib.IGilesUpload; -import edu.asu.diging.citesphere.user.IUser; - -public interface GilesUploadService { - - IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes); -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java deleted file mode 100644 index e63163c..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesUploadServiceImpl.java +++ /dev/null @@ -1,97 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.giles.impl; - -import javax.annotation.PostConstruct; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.ByteArrayResource; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.security.oauth2.common.OAuth2AccessToken; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.multipart.MultipartFile; - -import edu.asu.diging.citesphere.importer.core.service.giles.impl.UploadResponse; -import edu.asu.diging.citesphere.importer.core.service.giles.impl.GilesUploadServiceImpl.MultipartFileResource; -import edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager; -import edu.asu.diging.citesphere.importer.core.service.giles.GilesUploadService; -import edu.asu.diging.citesphere.model.bib.IGilesUpload; -import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; -import edu.asu.diging.citesphere.user.IUser; - -public class GilesUploadServiceImpl implements GilesUploadService{ - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private RestTemplate restTemplate; - - @Autowired - private InternalTokenManager internalTokenManager; - - @Value("${giles_baseurl}") - private String gilesBaseurl; - - @Value("${giles_upload_endpoint}") - private String uploadEndpoint; - - @PostConstruct - public void init() { - restTemplate = new RestTemplate(); - } - - /* - * (non-Javadoc) - * - * @see edu.asu.diging.citesphere.core.service.giles.impl.GilesUploadService# - * uploadFile(edu.asu.diging.citesphere.user.IUser, - * org.springframework.web.multipart.MultipartFile, byte[]) - */ - @Override - public IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes) { - - String token = getToken(user); - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - headers.setBearerAuth(token); - - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("files", new MultipartFileResource(fileBytes, file.getOriginalFilename())); - - HttpEntity> requestEntity = new HttpEntity<>(body, headers); - - ResponseEntity response = restTemplate.postForEntity(gilesBaseurl + uploadEndpoint, requestEntity, UploadResponse.class); - - IGilesUpload upload = new GilesUpload(); - upload.setProgressId(response.getBody().getId()); - upload.setUploadingUser(user.getUsername()); - return upload; - } - - private String getToken(IUser user) { - OAuth2AccessToken token = internalTokenManager.getAccessToken(user); - return token.getValue(); - } - - public class MultipartFileResource extends ByteArrayResource { - - private String filename; - - public MultipartFileResource(byte[] bytearray, String filename) { - super(bytearray); - this.filename = filename; - } - - @Override - public String getFilename() { - return this.filename; - } - } - -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java deleted file mode 100644 index 713fc16..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java +++ /dev/null @@ -1,11 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.oauth; - -import org.springframework.security.oauth2.common.OAuth2AccessToken; - -import edu.asu.diging.citesphere.user.IUser; - -public interface InternalTokenManager { - - OAuth2AccessToken getAccessToken(IUser user); - -} \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java deleted file mode 100644 index 52226da..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java +++ /dev/null @@ -1,121 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.oauth.impl; - -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Collection; -import java.util.Date; -import java.util.Optional; -import java.util.UUID; - -import javax.annotation.PostConstruct; -import javax.transaction.Transactional; -import javax.transaction.Transactional.TxType; - -import org.javers.common.collections.Sets; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.PropertySource; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.common.DefaultOAuth2AccessToken; -import org.springframework.security.oauth2.common.OAuth2AccessToken; -import org.springframework.security.oauth2.provider.AuthorizationRequest; -import org.springframework.security.oauth2.provider.ClientDetails; -import org.springframework.security.oauth2.provider.ClientDetailsService; -import org.springframework.security.oauth2.provider.OAuth2Authentication; -import org.springframework.security.oauth2.provider.OAuth2Request; -import org.springframework.security.oauth2.provider.OAuth2RequestFactory; -import org.springframework.security.oauth2.provider.TokenRequest; -import org.springframework.security.oauth2.provider.implicit.ImplicitTokenRequest; -import org.springframework.security.oauth2.provider.request.DefaultOAuth2RequestFactory; -import org.springframework.security.oauth2.provider.token.TokenStore; -import org.springframework.stereotype.Service; - -import edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager; -import edu.asu.diging.citesphere.importer.core.service.oauth.OAuthScope; -import edu.asu.diging.citesphere.user.IUser; - -@Service -@PropertySource({ "classpath:config.properties", "${appConfigFile:classpath:}/app.properties" }) -@SuppressWarnings("deprecation") -@Transactional -public class InternalTokenManagerImpl implements InternalTokenManager { - -// @Value("${_oauth_token_validity}") -// private Integer tokenValidity; -// -// @Value("${_citephere_oauth2_app_clientid}") -// private String citesphereClientId; -// -// @Autowired -// private TokenStore tokenStore; -// -// @Autowired -// private ClientDetailsService clientDetailsService; -// -// private OAuth2RequestFactory requestFactory; - -// @PostConstruct -// public void init() { -// this.requestFactory = new DefaultOAuth2RequestFactory(clientDetailsService); -// } - - /* (non-Javadoc) - * @see edu.asu.diging.citesphere.core.service.oauth.impl.InternalTokenManager#getAccessToken() - */ - @Override - @Transactional(TxType.REQUIRES_NEW) - public OAuth2AccessToken getAccessToken(IUser user) { -// Collection tokens = tokenStore.findTokensByClientIdAndUserName(citesphereClientId, user.getUsername()); -// Optional validToken = tokens.stream().filter(t -> !t.isExpired()).findFirst(); -// if (validToken.isPresent()) { -// return validToken.get(); -// } -// return createAccessToken(user); - return null; - } - -// private OAuth2AccessToken createAccessToken(IUser user) { -// DefaultOAuth2AccessToken token = new DefaultOAuth2AccessToken(UUID.randomUUID().toString()); -// if (tokenValidity > 0) { -// token.setExpiration(new Date(System.currentTimeMillis() + (tokenValidity * 1000L))); -// } -// token.setScope(Sets.asSet(OAuthScope.READ.getScope())); -// -// AuthorizationRequest request = new AuthorizationRequest(citesphereClientId, token.getScope()); -// TokenRequest implicitRequest = new ImplicitTokenRequest(requestFactory.createTokenRequest(request, "implicit"), requestFactory.createOAuth2Request(request)); -// -// OAuth2Authentication authentication = getOAuth2Authentication(clientDetailsService.loadClientByClientId(citesphereClientId), implicitRequest, user); -// tokenStore.storeAccessToken(token, authentication); -// System.out.println(extractTokenKey(token.getValue())); -// return token; -// } - -// private OAuth2Authentication getOAuth2Authentication(ClientDetails client, TokenRequest tokenRequest, IUser user) { -// OAuth2Request storedOAuth2Request = requestFactory.createOAuth2Request(client, tokenRequest); -// Authentication authentication = new UsernamePasswordAuthenticationToken(user, null, user.getRoles()); -// return new OAuth2Authentication(storedOAuth2Request, authentication); -// } -// -// private String extractTokenKey(String value) { -// if(value == null) { -// return null; -// } else { -// MessageDigest digest; -// try { -// digest = MessageDigest.getInstance("MD5"); -// } catch (NoSuchAlgorithmException var5) { -// throw new IllegalStateException("MD5 algorithm not available. Fatal (should be in the JDK)."); -// } -// -// try { -// byte[] e = digest.digest(value.getBytes("UTF-8")); -// return String.format("%032x", new Object[]{new BigInteger(1, e)}); -// } catch (UnsupportedEncodingException var4) { -// throw new IllegalStateException("UTF-8 encoding not available. Fatal (should be in the JDK)."); -// } -// } -// } -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 253548d..a1d6a7c 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -16,6 +16,7 @@ import org.springframework.social.zotero.api.Item; import org.springframework.social.zotero.api.Library; +import edu.asu.diging.citesphere.factory.impl.ParseExtra; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; @@ -147,6 +148,7 @@ private ContainerMeta parseJournalMeta(Map fields) { if(fields.get("issn") != null) { for(String issnString : fields.get("issn").split("and")) { Issn issn = new Issn(); + issn.setPubType("issn"); issn.setIssn(issnString.trim()); issnList.add(issn); } @@ -165,6 +167,9 @@ private ArticleMeta parseArticleMeta(Map fields) { data.setNote(fields.get("note")); item.setData(data); parseExtra.parseMetaDataNote(citation, item); + parseExtra.parseExtra(data, citation); + + System.out.println("citation ================================" + citation.toString()); meta.setArticleTitle(fields.get("title")); meta.setArticleShortTitle(fields.get("shorttitle")); @@ -211,8 +216,9 @@ private ArticleMeta parseArticleMeta(Map fields) { if(citation.getReferences() != null) { meta.setReferences(mapReferences(citation.getReferences())); + meta.setReferenceCount(meta.getReferences().size()+""); } - meta.setReferenceCount(meta.getReferences().size()+""); + // retrievalDate in note return meta; From 86078c50127e36f9d06cb0ff8c2d4c490248854b Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 1 Nov 2024 16:40:38 -0700 Subject: [PATCH 22/49] [CITE-163] Resolving parsing note issues --- .../core/service/parse/iterators/BibFileIterator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index a1d6a7c..9a150f3 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -49,6 +49,7 @@ public BibFileIterator(String filePath, String groupId) { this.filePath = filePath; this.groupId = groupId; parseExtra = new ParseExtra(); + parseExtra.init(); init(); } @@ -111,8 +112,8 @@ public BibEntry next() { // entry.setJournalMeta(parseJournalMeta(fields)); // entry.setArticleMeta(parseArticleMeta(fields)); // itemData = parseItemData(itemData, fields); - item.setData(itemData); - System.out.println("item ================= " +item); +// item.setData(itemData); +// System.out.println("item ================= " +item); // System.out.println("citation factory ===================== "+ citationFactory); // ICitation citation = citationFactory.createCitation(item, null); // convert citation to bibentry From d6305efd1c538e4f964fa39eeedd0be73a75f8b8 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 5 Nov 2024 12:02:34 -0700 Subject: [PATCH 23/49] [CITE-163] Resolving error with JSON parser --- .../core/service/parse/iterators/BibFileIterator.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 9a150f3..132b9f2 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -126,7 +126,7 @@ public BibEntry next() { String[] parts = line.split("=", 2); if (parts.length == 2) { String key = parts[0].trim(); - String value = parts[1].trim().replaceAll("[{},]", ""); // Remove curly braces and commas + String value = key.equals("annote") || key.equals("note")? parts[1].trim() : parts[1].trim().replaceAll("[{},]", ""); // Remove curly braces and commas fields.put(key, value); } } @@ -165,7 +165,13 @@ private ArticleMeta parseArticleMeta(Map fields) { ICitation citation = new Citation(); Item item = new Item(); Data data = new Data(); - data.setNote(fields.get("note")); + if(fields.containsKey("annote") && fields.get("annote").endsWith(",")) { + data.setNote(fields.get("annote").substring(0, fields.get("annote").length()-1)); + } + if(fields.containsKey("note") && fields.get("note").endsWith(",")) { + data.setExtra(fields.get("note").substring(0, fields.get("note").length()-1)); + } + item.setData(data); parseExtra.parseMetaDataNote(citation, item); parseExtra.parseExtra(data, citation); From 304cb178a5f635c7c0587c8ee7eabcf0cf576b58 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Wed, 6 Nov 2024 13:05:10 -0700 Subject: [PATCH 24/49] [CITE-163] Correctly parsed extra and note --- .../parse/iterators/BibFileIterator.java | 71 +++++++++++++++---- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 132b9f2..2494a50 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -22,6 +23,7 @@ import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; +import edu.asu.diging.citesphere.importer.core.model.impl.ContributionType; import edu.asu.diging.citesphere.importer.core.model.impl.Contributor; import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; @@ -29,9 +31,11 @@ import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.model.bib.IAffiliation; import edu.asu.diging.citesphere.model.bib.ICitation; +import edu.asu.diging.citesphere.model.bib.ICreator; import edu.asu.diging.citesphere.model.bib.IPerson; import edu.asu.diging.citesphere.model.bib.IReference; import edu.asu.diging.citesphere.model.bib.impl.Citation; +import edu.asu.diging.citesphere.model.bib.impl.Person; public class BibFileIterator implements BibEntryIterator { @@ -126,7 +130,10 @@ public BibEntry next() { String[] parts = line.split("=", 2); if (parts.length == 2) { String key = parts[0].trim(); - String value = key.equals("annote") || key.equals("note")? parts[1].trim() : parts[1].trim().replaceAll("[{},]", ""); // Remove curly braces and commas + String value = parts[1].trim(); + if (value.endsWith(",")) { + value = value.substring(0, value.length()-1).replaceAll("^\\{|\\}$", ""); + } fields.put(key, value); } } @@ -165,14 +172,48 @@ private ArticleMeta parseArticleMeta(Map fields) { ICitation citation = new Citation(); Item item = new Item(); Data data = new Data(); - if(fields.containsKey("annote") && fields.get("annote").endsWith(",")) { - data.setNote(fields.get("annote").substring(0, fields.get("annote").length()-1)); + if(fields.containsKey("annote")) { + data.setNote(fields.get("annote").replaceAll("\\\\", "").replaceAll("\\{textbackslash\\}n", "")); } - if(fields.containsKey("note") && fields.get("note").endsWith(",")) { - data.setExtra(fields.get("note").substring(0, fields.get("note").length()-1)); + if(fields.containsKey("note")) { + data.setExtra(fields.get("note").replace("\\\\", "").replaceAll("\\{textbackslash\\}n", "")); } item.setData(data); + + Set authors = new HashSet<>(); + String[] authorStringList = fields.get("author").split("and"); + for(String authorString: authorStringList) { + IPerson author = new Person(); + String[] authorParts = authorString.split(","); + author.setLastName(authorParts[0].trim()); + author.setFirstName(authorParts[1].trim()); + authors.add(author); + } + citation.setAuthors(authors); + + Set editors = new HashSet<>(); + String[] editorStringList = fields.get("author").split("and"); + for(String editorString: editorStringList) { + IPerson editor = new Person(); + String[] editorParts = editorString.split(","); + editor.setLastName(editorParts[0].trim()); + editor.setFirstName(editorParts[1].trim()); + editors.add(editor); + } + citation.setEditors(editors); + +// Add other creators + Set creators = new HashSet<>(); +// String[] creatorsStringList = fields.get("author").split("and"); +// for(String authorString: authorStringList) { +// IPerson author = new Person(); +// String[] authorParts = authorString.split(","); +// author.setLastName(authorParts[0].trim()); +// author.setFirstName(authorParts[1].trim()); +// } + citation.setOtherCreators(creators); + parseExtra.parseMetaDataNote(citation, item); parseExtra.parseExtra(data, citation); @@ -184,13 +225,13 @@ private ArticleMeta parseArticleMeta(Map fields) { List contributors = new ArrayList<>(); // List of authors if(citation.getAuthors() != null) { -// contributors.addAll(mapPersonToContributor(citation.getAuthors(), ContributionType.AUTHOR)); - // System.out.println(fields.get("author")+ "========================== authors"); + contributors.addAll(mapPersonToContributor(citation.getAuthors(), ContributionType.AUTHOR)); } // List of editors if(citation.getEditors() != null) { -// contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); + contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); } +// if(citation.getOtherCreators()) meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); ArticlePublicationDate publicationDate = new ArticlePublicationDate(); @@ -243,12 +284,14 @@ private List mapPersonToContributor(Set citationContributo contributor.setUri(person.getUri()); List affiliations = new ArrayList<>(); - for(IAffiliation institute: person.getAffiliations()) { - Affiliation affiliation = new Affiliation(); - affiliation.setName(institute.getName()); - affiliation.setUri(institute.getUri()); - affiliation.setLocalAuthorityId(institute.getLocalAuthorityId()); - affiliations.add(affiliation); + if(person.getAffiliations()!= null) { + for(IAffiliation institute: person.getAffiliations()) { + Affiliation affiliation = new Affiliation(); + affiliation.setName(institute.getName()); + affiliation.setUri(institute.getUri()); + affiliation.setLocalAuthorityId(institute.getLocalAuthorityId()); + affiliations.add(affiliation); + } } contributor.setAffiliations(affiliations); } From 74c0248c2a25dccc7af5afec6332fcce80520a94 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 7 Nov 2024 12:00:38 -0700 Subject: [PATCH 25/49] [CITE-163] Resolved authors & editors issue --- .../parse/iterators/BibFileIterator.java | 85 ++++++++----------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 2494a50..d67ce28 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -107,23 +107,12 @@ public BibEntry next() { Data itemData = new Data(); while (lineIterator.hasNext()) { String line = lineIterator.next().trim(); -// System.out.println("============ line - "+ line); if(!line.isBlank() && line.charAt(0)=='@') { - System.out.println("===================" + line.substring(1, line.indexOf('{'))); entry.setArticleType(typeMap.get(line.substring(1, line.indexOf('{')))); -// itemData.setItemType(typeMap.get(line.substring(line.indexOf('@')+1, line.indexOf('{')))); } else if (line.equals("}")) { - // entry.setJournalMeta(parseJournalMeta(fields)); - // entry.setArticleMeta(parseArticleMeta(fields)); -// itemData = parseItemData(itemData, fields); -// item.setData(itemData); -// System.out.println("item ================= " +item); -// System.out.println("citation factory ===================== "+ citationFactory); -// ICitation citation = citationFactory.createCitation(item, null); - // convert citation to bibentry entry.setJournalMeta(parseJournalMeta(fields)); entry.setArticleMeta(parseArticleMeta(fields)); - System.out.println("====================== entry - " + entry.toString()); +// System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; } else if (line.contains("=")) { @@ -178,47 +167,49 @@ private ArticleMeta parseArticleMeta(Map fields) { if(fields.containsKey("note")) { data.setExtra(fields.get("note").replace("\\\\", "").replaceAll("\\{textbackslash\\}n", "")); } - + item.setData(data); - + Set authors = new HashSet<>(); - String[] authorStringList = fields.get("author").split("and"); - for(String authorString: authorStringList) { - IPerson author = new Person(); - String[] authorParts = authorString.split(","); - author.setLastName(authorParts[0].trim()); - author.setFirstName(authorParts[1].trim()); - authors.add(author); + if (fields.containsKey("author")) { + String[] authorStringList = fields.get("author").split("and"); + for(String authorString: authorStringList) { + IPerson author = new Person(); + String[] authorParts = authorString.split(","); + author.setLastName(authorParts[0].trim()); + author.setFirstName(authorParts[1].trim()); + authors.add(author); + } } citation.setAuthors(authors); - + Set editors = new HashSet<>(); - String[] editorStringList = fields.get("author").split("and"); - for(String editorString: editorStringList) { - IPerson editor = new Person(); - String[] editorParts = editorString.split(","); - editor.setLastName(editorParts[0].trim()); - editor.setFirstName(editorParts[1].trim()); - editors.add(editor); + if (fields.containsKey("editor")) { + String[] editorStringList = fields.get("editor").split("and"); + for(String editorString: editorStringList) { + IPerson editor = new Person(); + String[] editorParts = editorString.split(","); + editor.setLastName(editorParts[0].trim()); + editor.setFirstName(editorParts[1].trim()); + editors.add(editor); + } } citation.setEditors(editors); - -// Add other creators + + // Add other creators Set creators = new HashSet<>(); -// String[] creatorsStringList = fields.get("author").split("and"); -// for(String authorString: authorStringList) { -// IPerson author = new Person(); -// String[] authorParts = authorString.split(","); -// author.setLastName(authorParts[0].trim()); -// author.setFirstName(authorParts[1].trim()); -// } + // String[] creatorsStringList = fields.get("author").split("and"); + // for(String authorString: authorStringList) { + // IPerson author = new Person(); + // String[] authorParts = authorString.split(","); + // author.setLastName(authorParts[0].trim()); + // author.setFirstName(authorParts[1].trim()); + // } citation.setOtherCreators(creators); - + parseExtra.parseMetaDataNote(citation, item); parseExtra.parseExtra(data, citation); - - System.out.println("citation ================================" + citation.toString()); - + meta.setArticleTitle(fields.get("title")); meta.setArticleShortTitle(fields.get("shorttitle")); // categoryGroups @@ -231,13 +222,10 @@ private ArticleMeta parseArticleMeta(Map fields) { if(citation.getEditors() != null) { contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); } -// if(citation.getOtherCreators()) + // if(citation.getOtherCreators()) meta.setContributors(contributors); // meta.setAuthorNotesCorrespondence(null); ArticlePublicationDate publicationDate = new ArticlePublicationDate(); - // if(citation != null) { - // publicationDate.setPublicationDate(dateParts.get(2).toString()); - // publicationDate.setPublicationMonth(dateParts.get(1).toString()); publicationDate.setPublicationYear(fields.get("year")); // } meta.setPublicationDate(publicationDate); @@ -266,14 +254,14 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setReferences(mapReferences(citation.getReferences())); meta.setReferenceCount(meta.getReferences().size()+""); } - + // retrievalDate in note + System.out.println("===================== meta - " + meta.toString()); return meta; } private List mapPersonToContributor(Set citationContributors, String contributionType) { - // String[] contributorStringList = contributorsString.split("and"); List contributors = new ArrayList(); for(IPerson person: citationContributors) { Contributor contributor = new Contributor(); @@ -294,6 +282,7 @@ private List mapPersonToContributor(Set citationContributo } } contributor.setAffiliations(affiliations); + contributors.add(contributor); } return contributors; } From 3f2f3fcbcb20ca34e32f1cff23c93a8bcb5d994e Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 8 Nov 2024 16:36:47 -0700 Subject: [PATCH 26/49] [CITE-163] Resolved DOI & ISBN parsing issue --- .../parse/iterators/BibFileIterator.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index d67ce28..e90ed12 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -11,15 +11,16 @@ import java.util.Set; import org.apache.commons.io.FileUtils; +import org.javers.common.collections.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.social.zotero.api.Data; import org.springframework.social.zotero.api.Item; -import org.springframework.social.zotero.api.Library; import edu.asu.diging.citesphere.factory.impl.ParseExtra; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; +import edu.asu.diging.citesphere.importer.core.model.impl.ArticleId; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; import edu.asu.diging.citesphere.importer.core.model.impl.ArticlePublicationDate; import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; @@ -46,7 +47,6 @@ public class BibFileIterator implements BibEntryIterator { private String filePath; private String groupId; private Iterator lineIterator; - private String currentLine = null; private Map typeMap; public BibFileIterator(String filePath, String groupId) { @@ -100,11 +100,6 @@ private void init() { public BibEntry next() { BibEntry entry = new Publication(); Map fields = new HashMap<>(); - Item item = new Item(); - Library lib = new Library(); - lib.setId(Long.parseLong(groupId)); - item.setLibrary(lib); - Data itemData = new Data(); while (lineIterator.hasNext()) { String line = lineIterator.next().trim(); if(!line.isBlank() && line.charAt(0)=='@') { @@ -112,7 +107,6 @@ public BibEntry next() { } else if (line.equals("}")) { entry.setJournalMeta(parseJournalMeta(fields)); entry.setArticleMeta(parseArticleMeta(fields)); -// System.out.println("====================== entry - " + entry.toString()); fields.clear(); break; } else if (line.contains("=")) { @@ -227,14 +221,23 @@ private ArticleMeta parseArticleMeta(Map fields) { // meta.setAuthorNotesCorrespondence(null); ArticlePublicationDate publicationDate = new ArticlePublicationDate(); publicationDate.setPublicationYear(fields.get("year")); - // } meta.setPublicationDate(publicationDate); meta.setVolume(fields.get("volume")); meta.setIssue(fields.get("number")); meta.setFirstPage(fields.get("pages").split("--")[0].trim()); meta.setLastPage(fields.get("pages").split("--")[1].trim()); meta.setSelfUri(fields.get("url")); - meta.setDoi(fields.get("doi")); + meta.setDoi(fields.get("doi")); + ArticleId doiId = new ArticleId(); + doiId.setPubIdType("doi"); + doiId.setId(fields.get("doi")); + ArticleId isbnId = new ArticleId(); + isbnId.setPubIdType("isbn"); + isbnId.setId(fields.get("isbn")); + List articleIds = new ArrayList<>(); + articleIds.add(doiId); + articleIds.add(isbnId); + meta.setArticleIds(articleIds); meta.setArticleAbstract(fields.get("abstract")); meta.setLanguage(fields.get("language")); // ReviewInfo review = new ReviewInfo(); @@ -257,7 +260,6 @@ private ArticleMeta parseArticleMeta(Map fields) { // retrievalDate in note - System.out.println("===================== meta - " + meta.toString()); return meta; } From 2b3a88836699873f300b87f9cd0c5bbb3b84a228 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 15 Nov 2024 16:37:53 -0700 Subject: [PATCH 27/49] [CITE-163] Added collectionId to JobInfo, so as to import to it --- .../importer/core/model/impl/ArticleMeta.java | 10 ++++++ .../importer/core/service/impl/JobInfo.java | 7 ++++ .../service/parse/impl/BibFileHandler.java | 3 +- .../parse/iterators/BibFileIterator.java | 33 ++++++++++++------- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index 74bc8e7..2e916ad 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -3,8 +3,11 @@ import java.util.ArrayList; import java.util.List; +import com.fasterxml.jackson.annotation.JsonProperty; + public class ArticleMeta { + private List collectionIds; private List articleIds; private String articleTitle; private String articleShortTitle; @@ -26,6 +29,7 @@ public class ArticleMeta { private String copyrightYear; private String copyrightHolder; private String selfUri; + @JsonProperty("DOI") private String doi; private String articleAbstract; private String language; @@ -46,6 +50,12 @@ public class ArticleMeta { private String referenceCount; private String retrievalDate; + public List getCollectionIds() { + return collectionIds; + } + public void setCollectionIds(List collectionIds) { + this.collectionIds = collectionIds; + } public List getArticleIds() { return articleIds; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java index 8eb8026..f267530 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java @@ -8,6 +8,7 @@ public class JobInfo { private String zotero; private String zoteroId; private String groupId; + private String collectionId; public String getZotero() { return zotero; @@ -27,5 +28,11 @@ public String getGroupId() { public void setGroupId(String groupId) { this.groupId = groupId; } + public String getCollectionId() { + return collectionId; + } + public void setCollectionId(String collectionId) { + this.collectionId = collectionId; + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index 2e42755..8abe1d1 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -24,8 +24,7 @@ public boolean canHandle(String path) throws HandlerTestException { @Override public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) throws IteratorCreationException { - System.out.println("inside bib file handler ................."); - return new BibFileIterator(path, info.getGroupId()); + return new BibFileIterator(path, info.getGroupId(), info.getCollectionId()); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index e90ed12..7235f5d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -36,6 +36,7 @@ import edu.asu.diging.citesphere.model.bib.IPerson; import edu.asu.diging.citesphere.model.bib.IReference; import edu.asu.diging.citesphere.model.bib.impl.Citation; +import edu.asu.diging.citesphere.model.bib.impl.Creator; import edu.asu.diging.citesphere.model.bib.impl.Person; public class BibFileIterator implements BibEntryIterator { @@ -46,12 +47,14 @@ public class BibFileIterator implements BibEntryIterator { private String filePath; private String groupId; + private String collectionId; private Iterator lineIterator; private Map typeMap; - public BibFileIterator(String filePath, String groupId) { + public BibFileIterator(String filePath, String groupId, String collectionId) { this.filePath = filePath; this.groupId = groupId; + this.collectionId = collectionId; parseExtra = new ParseExtra(); parseExtra.init(); init(); @@ -161,7 +164,6 @@ private ArticleMeta parseArticleMeta(Map fields) { if(fields.containsKey("note")) { data.setExtra(fields.get("note").replace("\\\\", "").replaceAll("\\{textbackslash\\}n", "")); } - item.setData(data); Set authors = new HashSet<>(); @@ -192,18 +194,28 @@ private ArticleMeta parseArticleMeta(Map fields) { // Add other creators Set creators = new HashSet<>(); - // String[] creatorsStringList = fields.get("author").split("and"); - // for(String authorString: authorStringList) { - // IPerson author = new Person(); - // String[] authorParts = authorString.split(","); - // author.setLastName(authorParts[0].trim()); - // author.setFirstName(authorParts[1].trim()); - // } +// if(fields.containsKey("editor")) +// String[] artistStringList = fields.get("artist").split("and"); +// for(String artistString: artistStringList) { +// ICreator artist = new Creator(); +// IPerson person = new Person(); +// String[] authorParts = artistString.split(","); +// person.setLastName(authorParts[0].trim()); +// person.setFirstName(authorParts[1].trim()); +// artist.setPerson(person); +// artist.setRole("Artist"); +// creators.add(artist); +// } citation.setOtherCreators(creators); parseExtra.parseMetaDataNote(citation, item); parseExtra.parseExtra(data, citation); + List collectionIds = new ArrayList<>(); + if (collectionId != null && !collectionId.trim().isEmpty()) { + collectionIds.add(collectionId); + } + meta.setCollectionIds(collectionIds); meta.setArticleTitle(fields.get("title")); meta.setArticleShortTitle(fields.get("shorttitle")); // categoryGroups @@ -257,9 +269,6 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setReferences(mapReferences(citation.getReferences())); meta.setReferenceCount(meta.getReferences().size()+""); } - - - // retrievalDate in note return meta; } From fb3fb21a6b5f57f843ba32c168fcddb1ad0ca11e Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Mon, 18 Nov 2024 16:08:22 -0700 Subject: [PATCH 28/49] [CITE-163] Cleaned code for bibFileIterator --- .../parse/iterators/BibFileIterator.java | 121 +++++++----------- 1 file changed, 44 insertions(+), 77 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 7235f5d..e9f7ad9 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -3,6 +3,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -11,7 +12,6 @@ import java.util.Set; import org.apache.commons.io.FileUtils; -import org.javers.common.collections.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.social.zotero.api.Data; @@ -36,7 +36,6 @@ import edu.asu.diging.citesphere.model.bib.IPerson; import edu.asu.diging.citesphere.model.bib.IReference; import edu.asu.diging.citesphere.model.bib.impl.Citation; -import edu.asu.diging.citesphere.model.bib.impl.Creator; import edu.asu.diging.citesphere.model.bib.impl.Person; public class BibFileIterator implements BibEntryIterator { @@ -71,33 +70,12 @@ private void init() { typeMap.put("book", Publication.BOOK); typeMap.put("book-chapter", Publication.BOOK_CHAPTER); typeMap.put("monograph", Publication.BOOK); - // typeMap.put("journal-issue", Publication.JOURNAL_ISSUE); - // typeMap.put("reference-entry", Publication.REFERNCE_ENTRY); - // typeMap.put("posted-content", Publication.POSTED_CONTENT); - // typeMap.put("component", Publication.COMPONENT); - // typeMap.put("edited-book", Publication.EDITED_BOOK); typeMap.put("proceedings-article", Publication.PROCEEDINGS_PAPER); - // typeMap.put("dissertation", Publication.DISSERTATION); typeMap.put("book-section", Publication.BOOK_CHAPTER); - // typeMap.put("report-component", Publication.REPORT_COMPONENT); - // typeMap.put("report", Publication.REPORT); - // typeMap.put("peer-review", Publication.PEER_REVIEW); - // typeMap.put("book-track", Publication.BOOK_TRACK); - // typeMap.put("book-part", Publication.BOOK_PART); - // typeMap.put("other", Publication.OTHER); - // typeMap.put("journal-volume", Publication.JORUNAL_VOLUME); - // typeMap.put("book-set", Publication.BOOK_SET); - // typeMap.put("journal", Publication.JOURNAL); - // typeMap.put("proceedings-series", Publication.PROCEEDINGS_SERIES); - // typeMap.put("report-series", Publication.REPORT_SERIES); - // typeMap.put("proceedings", Publication.PROCEEDINGS); - // typeMap.put("database", Publication.DATABASE); - // typeMap.put("standard", Publication.STANDARD); - // typeMap.put("reference-book", Publication.REFERENCE_BOOK); - // typeMap.put("grant", Publication.GRANT); - // typeMap.put("dataset", Publication.DATASET); - // typeMap.put("book-series", Publication.BOOK_SERIES); - } + typeMap.put("research-article", Publication.ARTICLE); + typeMap.put("book-review", Publication.REVIEW); + typeMap.put("patent", Publication.PROCEEDINGS_PAPER); + } @Override public BibEntry next() { @@ -130,14 +108,12 @@ public BibEntry next() { private ContainerMeta parseJournalMeta(Map fields) { ContainerMeta meta = new ContainerMeta(); - // journalIds - // meta.setContainerTitle(fields.get("title")); + List journalAbbrev = new ArrayList<>(); journalAbbrev.add(fields.get("journal")); meta.setJournalAbbreviations(journalAbbrev); meta.setPublisherName(fields.get("publisher")); meta.setPublisherLocation(fields.get("place")); - // publisherAddress List issnList = new ArrayList(); if(fields.get("issn") != null) { for(String issnString : fields.get("issn").split("and")) { @@ -149,7 +125,6 @@ private ContainerMeta parseJournalMeta(Map fields) { } meta.setIssns(issnList); meta.setSeriesTitle(fields.get("series")); - // private String seriesSubTitle; return meta; } @@ -192,20 +167,8 @@ private ArticleMeta parseArticleMeta(Map fields) { } citation.setEditors(editors); - // Add other creators + Set creators = new HashSet<>(); -// if(fields.containsKey("editor")) -// String[] artistStringList = fields.get("artist").split("and"); -// for(String artistString: artistStringList) { -// ICreator artist = new Creator(); -// IPerson person = new Person(); -// String[] authorParts = artistString.split(","); -// person.setLastName(authorParts[0].trim()); -// person.setFirstName(authorParts[1].trim()); -// artist.setPerson(person); -// artist.setRole("Artist"); -// creators.add(artist); -// } citation.setOtherCreators(creators); parseExtra.parseMetaDataNote(citation, item); @@ -218,7 +181,7 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setCollectionIds(collectionIds); meta.setArticleTitle(fields.get("title")); meta.setArticleShortTitle(fields.get("shorttitle")); - // categoryGroups + List contributors = new ArrayList<>(); // List of authors if(citation.getAuthors() != null) { @@ -228,9 +191,12 @@ private ArticleMeta parseArticleMeta(Map fields) { if(citation.getEditors() != null) { contributors.addAll(mapPersonToContributor(citation.getEditors(), ContributionType.EDITOR)); } - // if(citation.getOtherCreators()) + // List of other creators + if(citation.getOtherCreators() != null) { + contributors.addAll(mapCreatorToContributor(citation.getOtherCreators())); + + } meta.setContributors(contributors); - // meta.setAuthorNotesCorrespondence(null); ArticlePublicationDate publicationDate = new ArticlePublicationDate(); publicationDate.setPublicationYear(fields.get("year")); meta.setPublicationDate(publicationDate); @@ -252,18 +218,6 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setArticleIds(articleIds); meta.setArticleAbstract(fields.get("abstract")); meta.setLanguage(fields.get("language")); - // ReviewInfo review = new ReviewInfo(); - // if (item.getReview() != null) { - // review.setFullDescription(item.getReview().getCompetingInterestStatement()); - // } - // meta.setReviewInfo(review); - // meta.setDocumentType(fields.get("type")); // document type in note - // conferenceInfo in note - map to conferenceTitle, conferenceDate, conferenceLocation, conferenceSponsor, conferenceHost - // keywords in note - // reprintAddress in note - // additionalData in note - // funding in note - map to fundingInfo, fundingText - if(citation.getReferences() != null) { meta.setReferences(mapReferences(citation.getReferences())); @@ -275,28 +229,41 @@ private ArticleMeta parseArticleMeta(Map fields) { private List mapPersonToContributor(Set citationContributors, String contributionType) { List contributors = new ArrayList(); for(IPerson person: citationContributors) { - Contributor contributor = new Contributor(); - contributor.setContributionType(contributionType); - contributor.setGivenName(person.getFirstName()); - contributor.setSurname(person.getLastName()); - contributor.setFullName(person.getName()); - contributor.setUri(person.getUri()); - - List affiliations = new ArrayList<>(); - if(person.getAffiliations()!= null) { - for(IAffiliation institute: person.getAffiliations()) { - Affiliation affiliation = new Affiliation(); - affiliation.setName(institute.getName()); - affiliation.setUri(institute.getUri()); - affiliation.setLocalAuthorityId(institute.getLocalAuthorityId()); - affiliations.add(affiliation); - } - } - contributor.setAffiliations(affiliations); + Contributor contributor = mapSinglePerson(person, contributionType); contributors.add(contributor); } return contributors; } + + private Contributor mapSinglePerson(IPerson person, String contributionType) { + Contributor contributor = new Contributor(); + contributor.setContributionType(contributionType); + contributor.setGivenName(person.getFirstName()); + contributor.setSurname(person.getLastName()); + contributor.setFullName(person.getName()); + contributor.setUri(person.getUri()); + + List affiliations = new ArrayList<>(); + if(person.getAffiliations()!= null) { + for(IAffiliation institute: person.getAffiliations()) { + Affiliation affiliation = new Affiliation(); + affiliation.setName(institute.getName()); + affiliation.setUri(institute.getUri()); + affiliation.setLocalAuthorityId(institute.getLocalAuthorityId()); + affiliations.add(affiliation); + } + } + contributor.setAffiliations(affiliations); + return contributor; + } + + private List mapCreatorToContributor(Set creators) { + List contributors = new ArrayList(); + for(ICreator creator: creators) { + contributors.add(mapSinglePerson(creator.getPerson(), creator.getRole())); + } + return contributors; + } private List mapReferences(Set citationReferences) { List references = new ArrayList(); From 35075f9bbcfedab94aa7ec0e3e015e01bca896c7 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 19 Nov 2024 12:05:49 -0700 Subject: [PATCH 29/49] [CITE-163] Resolving issue with CollectionIds --- .../importer/core/model/impl/ArticleMeta.java | 10 +++++----- .../service/parse/iterators/BibFileIterator.java | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index 2e916ad..7d8102b 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -7,7 +7,7 @@ public class ArticleMeta { - private List collectionIds; + private List collections; private List articleIds; private String articleTitle; private String articleShortTitle; @@ -50,11 +50,11 @@ public class ArticleMeta { private String referenceCount; private String retrievalDate; - public List getCollectionIds() { - return collectionIds; + public List getCollections() { + return collections; } - public void setCollectionIds(List collectionIds) { - this.collectionIds = collectionIds; + public void setCollections(List collections) { + this.collections = collections; } public List getArticleIds() { return articleIds; diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index e9f7ad9..1a4e79d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -109,9 +109,11 @@ public BibEntry next() { private ContainerMeta parseJournalMeta(Map fields) { ContainerMeta meta = new ContainerMeta(); - List journalAbbrev = new ArrayList<>(); - journalAbbrev.add(fields.get("journal")); - meta.setJournalAbbreviations(journalAbbrev); + if(fields.containsKey("journal")) { + List journalAbbrev = new ArrayList<>(); + journalAbbrev.add(fields.get("journal")); + meta.setJournalAbbreviations(journalAbbrev); + } meta.setPublisherName(fields.get("publisher")); meta.setPublisherLocation(fields.get("place")); List issnList = new ArrayList(); @@ -178,7 +180,7 @@ private ArticleMeta parseArticleMeta(Map fields) { if (collectionId != null && !collectionId.trim().isEmpty()) { collectionIds.add(collectionId); } - meta.setCollectionIds(collectionIds); + meta.setCollections(collectionIds); meta.setArticleTitle(fields.get("title")); meta.setArticleShortTitle(fields.get("shorttitle")); @@ -202,8 +204,10 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setPublicationDate(publicationDate); meta.setVolume(fields.get("volume")); meta.setIssue(fields.get("number")); - meta.setFirstPage(fields.get("pages").split("--")[0].trim()); - meta.setLastPage(fields.get("pages").split("--")[1].trim()); + if(fields.containsKey("pages")) { + meta.setFirstPage(fields.get("pages").split("--")[0].trim()); + meta.setLastPage(fields.get("pages").split("--")[1].trim()); + } meta.setSelfUri(fields.get("url")); meta.setDoi(fields.get("doi")); ArticleId doiId = new ArticleId(); From 604a621edbaa4d4932444ba854e213a89005e981 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Wed, 20 Nov 2024 13:03:45 -0700 Subject: [PATCH 30/49] [CITE-163] Resolved importing to collection issue --- .../importer/core/service/parse/iterators/BibFileIterator.java | 2 +- .../importer/core/zotero/template/ItemJsonGenerator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 1a4e79d..265380b 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -75,7 +75,7 @@ private void init() { typeMap.put("research-article", Publication.ARTICLE); typeMap.put("book-review", Publication.REVIEW); typeMap.put("patent", Publication.PROCEEDINGS_PAPER); - } + } @Override public BibEntry next() { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java index 2d9240a..5915da0 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java @@ -106,7 +106,7 @@ public JsonNode processRelations(JsonNode node, BibEntry bibEntry) { } public JsonNode processCollections(JsonNode node, BibEntry bibEntry) { - return mapper.createArrayNode(); + return mapper.valueToTree(bibEntry.getArticleMeta().getCollections()); } protected ObjectMapper getObjectMapper() { From e9cbdbe1e92853598cd9ad5fc170986f852d5c65 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Wed, 27 Nov 2024 13:02:41 -0700 Subject: [PATCH 31/49] [CITE-163] Added file path to Article Meta --- .../citesphere/importer/core/model/impl/ArticleMeta.java | 7 +++++++ .../core/service/parse/iterators/BibFileIterator.java | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index 7d8102b..38432f3 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -35,6 +35,7 @@ public class ArticleMeta { private String language; private ReviewInfo reviewInfo; private String documentType; + private String filePath; private String conferenceTitle; private String conferenceDate; private String conferenceLocation; @@ -212,6 +213,12 @@ public String getDocumentType() { public void setDocumentType(String documentType) { this.documentType = documentType; } + public String getFilePath() { + return filePath; + } + public void setFilePath(String filePath) { + this.filePath = filePath; + } public String getConferenceTitle() { return conferenceTitle; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 265380b..c500817 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -227,6 +227,12 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setReferences(mapReferences(citation.getReferences())); meta.setReferenceCount(meta.getReferences().size()+""); } + + if(fields.containsKey("file")) { + String[] fileParts = fields.get("file").split(":"); + meta.setDocumentType(fileParts[2]); + meta.setFilePath(fileParts[1]); + } return meta; } From bb79bdde88f750cfe2483b2b68c8aaa4dd2cfabe Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Mon, 2 Dec 2024 13:02:32 -0700 Subject: [PATCH 32/49] [CITE-163] Added method to CitesphereConnector to getItem --- .../core/service/ICitesphereConnector.java | 2 ++ .../service/impl/CitesphereConnector.java | 28 +++++++++++++++++++ .../impl/CollectionImportProcessor.java | 13 +++++++++ .../src/main/resources/config.properties | 1 + 4 files changed, 44 insertions(+) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java index 0f47939..e9bfaa4 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java @@ -8,5 +8,7 @@ public interface ICitesphereConnector { JobInfo getJobInfo(String apiToken) throws CitesphereCommunicationException; String getUploadeFile(String apiToken) throws CitesphereCommunicationException; + + String getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException; } \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java index 93430ac..b9ffe0f 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java @@ -72,6 +72,9 @@ public class CitesphereConnector implements ICitesphereConnector { @Value("${_citesphere_job_info_path}") private String jobInfoPath; + @Value("${_citesphere_get_item_path}") + private String getItemPath; + private RestTemplate restTemplate; private String accessToken; @@ -192,6 +195,31 @@ public void doWithRequest(ClientHttpRequest request) throws IOException { return filepath; } + @Override + public String getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException { + String path = getItemPath.replace("{groupId}", groupId).replace("{item}", itemKey); + System.out.println("path ====================================" + path); + @SuppressWarnings("unchecked") + ResponseEntity response = (ResponseEntity) makeApiCall(path, apiToken, String.class); + HttpStatus status = response.getStatusCode(); + +// JobInfo info = null; + if (status == HttpStatus.OK) { + String responseBody = response.getBody(); + System.out.println("ITem ============================" + responseBody); + ObjectMapper mapper = new ObjectMapper(); +// try { +// info = mapper.readValue(responseBody, JobInfo.class); +// } catch (IOException e) { +// throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); +// } + } else { + throw new CitesphereCommunicationException("Could not communicate with Citesphere properly. Got " + status); + } + + return response.getBody(); + } + private ResponseEntity makeApiCall(String url, String apiToken, Class responseType) throws CitesphereCommunicationException { HttpEntity entity = buildHeaders(apiToken); ResponseEntity response; diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index 3b9c67c..f4d4764 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -3,6 +3,7 @@ import java.net.URISyntaxException; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import javax.annotation.PostConstruct; @@ -135,6 +136,18 @@ public void process(KafkaJobMessage message) { ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); + response.getSuccessful().forEach((key, value) -> { + System.out.println("=========================================================="); + System.out.println("Key -" + key.toString()); + System.out.println("value - "+ value.toString()); + + try { + String item = connector.getItem(message.getId(), info.getGroupId(), value.toString()); + } catch (CitesphereCommunicationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }); } response = response != null ? response : new ItemCreationResponse(); diff --git a/citesphere-importer/src/main/resources/config.properties b/citesphere-importer/src/main/resources/config.properties index b19b7ab..9e717eb 100644 --- a/citesphere-importer/src/main/resources/config.properties +++ b/citesphere-importer/src/main/resources/config.properties @@ -9,6 +9,7 @@ _citesphere_download_path=${citesphere.download.path} _citesphere_token_endpoint=api/oauth/token?grant_type=client_credentials _citesphere_upload_path=api/v1/upload _citesphere_job_info_path=api/v1/job/info +_citesphere_get_item_path=api/v1/groups/{groupId}/items/{item} _zotero_client_key=${zotero.client.key} _zotero_client_secret=${zotero.client.secret} From b51c18c0edc04ca5547984e551b6ec0cb8b94b43 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 24 Jan 2025 16:28:12 -0700 Subject: [PATCH 33/49] [CITE-163] Added uploadFile & makePostApiCall to CitesphereConnector --- .../core/service/ICitesphereConnector.java | 6 +- .../service/impl/CitesphereConnector.java | 84 +++++++++++++++---- .../impl/CollectionImportProcessor.java | 63 +++++++++++--- .../src/main/resources/config.properties | 1 + 4 files changed, 127 insertions(+), 27 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java index e9bfaa4..2f1528d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java @@ -1,5 +1,8 @@ package edu.asu.diging.citesphere.importer.core.service; +import org.springframework.social.zotero.api.Item; +import org.springframework.web.multipart.MultipartFile; + import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; @@ -9,6 +12,7 @@ public interface ICitesphereConnector { String getUploadeFile(String apiToken) throws CitesphereCommunicationException; - String getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException; + Item getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException; + String uploadFile(String apiToken, String groupId, String itemKey, MultipartFile[] files) throws CitesphereCommunicationException; } \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java index b9ffe0f..6a16829 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java @@ -9,13 +9,11 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.UUID; import javax.annotation.PostConstruct; -import javax.print.attribute.standard.DateTimeAtCompleted; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; @@ -29,6 +27,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.http.client.ClientHttpRequest; import org.springframework.http.client.ClientHttpResponse; +import org.springframework.social.zotero.api.Item; import org.springframework.stereotype.Service; import org.springframework.web.client.DefaultResponseErrorHandler; import org.springframework.web.client.RequestCallback; @@ -36,6 +35,7 @@ import org.springframework.web.client.ResponseExtractor; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; import org.springframework.web.util.DefaultUriBuilderFactory; import com.fasterxml.jackson.databind.JsonNode; @@ -75,6 +75,9 @@ public class CitesphereConnector implements ICitesphereConnector { @Value("${_citesphere_get_item_path}") private String getItemPath; + @Value("${_citesphere_upload_file_path}") + private String uploadFilePath; + private RestTemplate restTemplate; private String accessToken; @@ -196,32 +199,30 @@ public void doWithRequest(ClientHttpRequest request) throws IOException { } @Override - public String getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException { + public Item getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException { String path = getItemPath.replace("{groupId}", groupId).replace("{item}", itemKey); - System.out.println("path ====================================" + path); @SuppressWarnings("unchecked") ResponseEntity response = (ResponseEntity) makeApiCall(path, apiToken, String.class); HttpStatus status = response.getStatusCode(); -// JobInfo info = null; + Item item = null; if (status == HttpStatus.OK) { String responseBody = response.getBody(); - System.out.println("ITem ============================" + responseBody); ObjectMapper mapper = new ObjectMapper(); -// try { -// info = mapper.readValue(responseBody, JobInfo.class); -// } catch (IOException e) { -// throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); -// } + try { + item = mapper.readValue(responseBody, Item.class); + } catch (IOException e) { + throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); + } } else { throw new CitesphereCommunicationException("Could not communicate with Citesphere properly. Got " + status); } - return response.getBody(); + return item; } private ResponseEntity makeApiCall(String url, String apiToken, Class responseType) throws CitesphereCommunicationException { - HttpEntity entity = buildHeaders(apiToken); + HttpEntity entity = buildHeaders(apiToken, "body"); ResponseEntity response; try { response = restTemplate.exchange(url, HttpMethod.GET, entity, responseType); @@ -237,13 +238,36 @@ private ResponseEntity makeApiCall(String url, String apiToken, Class resp } // let's try again after getting a new OAuth token - entity = buildHeaders(apiToken); + entity = buildHeaders(apiToken, "body"); + response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class); + } + return response; + } + + private ResponseEntity makePostApiCall(String url, String apiToken, Object requestBody, Class responseType) throws CitesphereCommunicationException { + HttpEntity entity = buildHeaders(apiToken, requestBody); + ResponseEntity response; + try { + response = restTemplate.exchange(url, HttpMethod.POST, entity, responseType); + } catch (RestClientException ex) { + throw new CitesphereCommunicationException("Could not understand server.", ex); + } + HttpStatus status = response.getStatusCode(); + + if (status == HttpStatus.UNAUTHORIZED) { + String responseBody = response.getBody().toString(); + if (!refreshToken(responseBody)) { + throw new CitesphereCommunicationException("Could not understand returned error message: " + responseBody); + } + + // let's try again after getting a new OAuth token + entity = buildHeaders(apiToken, "body"); response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class); } return response; } - private HttpEntity buildHeaders(String apiToken) { + private HttpEntity buildHeaders(String apiToken, Object requestBody) { if (accessToken == null) { accessToken = getAccessToken(); } @@ -251,7 +275,7 @@ private HttpEntity buildHeaders(String apiToken) { HttpHeaders headers = new HttpHeaders(); headers.setBearerAuth(accessToken); headers.put(CitesphereHeaders.CITESPHERE_API_TOKEN, Arrays.asList(apiToken)); - HttpEntity entity = new HttpEntity<>("body", headers); + HttpEntity entity = new HttpEntity<>(requestBody, headers); return entity; } @@ -292,4 +316,32 @@ private boolean refreshToken(String errorMessage) { return false; } + + @Override + public String uploadFile(String apiToken, String groupId, String itemKey, MultipartFile[] files) throws CitesphereCommunicationException { + String path = uploadFilePath.replace("{groupId}", groupId).replace("{item}", itemKey); + + Map requestBody = new HashMap<>(); + requestBody.put("files", files); + + @SuppressWarnings("unchecked") + ResponseEntity response = (ResponseEntity) makePostApiCall(path, apiToken, requestBody, String.class); + HttpStatus status = response.getStatusCode(); + +// Item item = null; + if (status == HttpStatus.OK) { + String responseBody = response.getBody(); + System.out.println("=========================== response - " + responseBody); + ObjectMapper mapper = new ObjectMapper(); +// try { +// item = mapper.readValue(responseBody, Item.class); +// } catch (IOException e) { +// throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); +// } + } else { + throw new CitesphereCommunicationException("Could not communicate with Citesphere properly. Got " + status); + } + + return response.getBody(); + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index f4d4764..0e16463 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -1,7 +1,12 @@ package edu.asu.diging.citesphere.importer.core.service.impl; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -11,6 +16,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -136,18 +142,35 @@ public void process(KafkaJobMessage message) { ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); - response.getSuccessful().forEach((key, value) -> { + for(int i=0; i entry = (Entry) response.getSuccessful().get(i); System.out.println("=========================================================="); - System.out.println("Key -" + key.toString()); - System.out.println("value - "+ value.toString()); - + System.out.println("Key -" + entry.getKey()); + System.out.println("value - "+ entry.getValue()); try { - String item = connector.getItem(message.getId(), info.getGroupId(), value.toString()); - } catch (CitesphereCommunicationException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - }); +// Item item = connector.getItem(message.getId(), info.getGroupId(), value.toString().trim()); +// System.out.println(item.toString()+ " ============================= item"); + String res = connector.uploadFile(message.getId(), info.getGroupId(), entry.getValue().trim(), new File(root.get(i).get("filePath").asText())); + } catch (CitesphereCommunicationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + +// response.getSuccessful().forEach((key, value) -> { +// System.out.println("=========================================================="); +// System.out.println("Key -" + key.toString()); +// System.out.println("value - "+ value.toString()); +// +// try { +//// Item item = connector.getItem(message.getId(), info.getGroupId(), value.toString().trim()); +//// System.out.println(item.toString()+ " ============================= item"); +// String res = connector.uploadFile(message.getId(), info.getGroupId(), value.toString().trim(), root.get(i).get("filePath")); +// } catch (CitesphereCommunicationException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// }); } response = response != null ? response : new ItemCreationResponse(); @@ -155,6 +178,26 @@ public void process(KafkaJobMessage message) { } + public static MultipartFile[] getMultipartFilesFromPaths(List filePaths) throws IOException { + List multipartFiles = new ArrayList<>(); + + for (String filePath : filePaths) { + File file = new File(filePath); + + try (FileInputStream inputStream = new FileInputStream(file)) { + MultipartFile multipartFile = new MockMultipartFile( + file.getName(), // Original file name + file.getName(), // File name for upload + "application/octet-stream", // Content type (default to binary) + inputStream // Input stream of the file + ); + multipartFiles.add(multipartFile); + } + } + + return multipartFiles.toArray(new MultipartFile[0]); + } + private void sendMessage(ItemCreationResponse message, String jobId, Status status, ResponseCode code) { KafkaImportReturnMessage returnMessage = new KafkaImportReturnMessage(message, jobId); returnMessage.setStatus(status); diff --git a/citesphere-importer/src/main/resources/config.properties b/citesphere-importer/src/main/resources/config.properties index 9e717eb..543fe4e 100644 --- a/citesphere-importer/src/main/resources/config.properties +++ b/citesphere-importer/src/main/resources/config.properties @@ -10,6 +10,7 @@ _citesphere_token_endpoint=api/oauth/token?grant_type=client_credentials _citesphere_upload_path=api/v1/upload _citesphere_job_info_path=api/v1/job/info _citesphere_get_item_path=api/v1/groups/{groupId}/items/{item} +_citesphere_upload_file_path=api/v1/groups/{groupId}/items/{item}/file _zotero_client_key=${zotero.client.key} _zotero_client_secret=${zotero.client.secret} From ab8fb239453ed7e60ffc607c846eeefa21dae1ce Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 28 Jan 2025 16:32:37 -0700 Subject: [PATCH 34/49] [CITE-163] Added Giles connector & supporting classes --- .../core/service/IGilesConnector.java | 11 ++ .../service/giles/impl/GilesConnector.java | 88 +++++++++++++ .../impl/CollectionImportProcessor.java | 27 ++-- .../importer/core/service/impl/JobInfo.java | 7 + .../service/oauth/InternalTokenManager.java | 12 ++ .../oauth/impl/InternalTokenManagerImpl.java | 120 ++++++++++++++++++ 6 files changed, 257 insertions(+), 8 deletions(-) create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java new file mode 100644 index 0000000..be910cd --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java @@ -0,0 +1,11 @@ +package edu.asu.diging.citesphere.importer.core.service; + +import org.springframework.web.multipart.MultipartFile; + +import edu.asu.diging.citesphere.model.bib.IGilesUpload; +import edu.asu.diging.citesphere.user.IUser; + +public interface IGilesConnector { + + IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes); +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java new file mode 100644 index 0000000..9adbb59 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java @@ -0,0 +1,88 @@ +package edu.asu.diging.citesphere.importer.core.service.giles.impl; + +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; +import edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; +import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; +import edu.asu.diging.citesphere.user.IUser; + +public class GilesConnector implements IGilesConnector { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private RestTemplate restTemplate; + + @Autowired + private InternalTokenManager internalTokenManager; + + @Value("${giles_baseurl}") + private String gilesBaseurl; + + @Value("${giles_upload_endpoint}") + private String uploadEndpoint; + + @PostConstruct + public void init() { + restTemplate = new RestTemplate(); + } + + @Override + public IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes) { + + String token = getToken(user); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.setBearerAuth(token); + + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("files", new MultipartFileResource(fileBytes, file.getOriginalFilename())); + + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + + ResponseEntity response = restTemplate.postForEntity(gilesBaseurl + uploadEndpoint, requestEntity, UploadResponse.class); + + IGilesUpload upload = new GilesUpload(); + upload.setProgressId(response.getBody().getId()); + upload.setUploadingUser(user.getUsername()); + return upload; + } + + private String getToken(IUser user) { + OAuth2AccessToken token = internalTokenManager.getAccessToken(user); + return token.getValue(); + } + + public class MultipartFileResource extends ByteArrayResource { + + private String filename; + + public MultipartFileResource(byte[] bytearray, String filename) { + super(bytearray); + this.filename = filename; + } + + @Override + public String getFilename() { + return this.filename; + } + } + +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index 0e16463..822080a 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import javax.annotation.PostConstruct; @@ -32,6 +33,7 @@ import edu.asu.diging.citesphere.importer.core.model.ItemType; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.service.ICitesphereConnector; +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; import edu.asu.diging.citesphere.importer.core.service.IImportProcessor; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; @@ -43,6 +45,9 @@ import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; import edu.asu.diging.citesphere.messages.model.ResponseCode; import edu.asu.diging.citesphere.messages.model.Status; +import edu.asu.diging.citesphere.user.IUser; +import edu.asu.diging.simpleusers.core.data.UserRepository; +import edu.asu.diging.simpleusers.core.model.impl.User; @Service public class CollectionImportProcessor implements IImportProcessor { @@ -64,6 +69,12 @@ public class CollectionImportProcessor implements IImportProcessor { @Autowired private KafkaRequestProducer requestProducer; + @Autowired + private IGilesConnector gilesConnector; + + @Autowired + private UserRepository userRepository; + private Map itemTypeMapping = new HashMap<>(); @PostConstruct @@ -147,14 +158,14 @@ public void process(KafkaJobMessage message) { System.out.println("=========================================================="); System.out.println("Key -" + entry.getKey()); System.out.println("value - "+ entry.getValue()); - try { -// Item item = connector.getItem(message.getId(), info.getGroupId(), value.toString().trim()); -// System.out.println(item.toString()+ " ============================= item"); - String res = connector.uploadFile(message.getId(), info.getGroupId(), entry.getValue().trim(), new File(root.get(i).get("filePath").asText())); - } catch (CitesphereCommunicationException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + IUser user = null; + + Optional foundUser = userRepository.findById(info.getUsername()); + if (foundUser.isPresent()) { + user = (IUser) foundUser.get(); + } + + } // response.getSuccessful().forEach((key, value) -> { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java index f267530..3412f7a 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java @@ -9,6 +9,7 @@ public class JobInfo { private String zoteroId; private String groupId; private String collectionId; + private String username; public String getZotero() { return zotero; @@ -34,5 +35,11 @@ public String getCollectionId() { public void setCollectionId(String collectionId) { this.collectionId = collectionId; } + public String getUsername() { + return username; + } + public void setUsername(String username) { + this.username = username; + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java new file mode 100644 index 0000000..ea263b0 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java @@ -0,0 +1,12 @@ +package edu.asu.diging.citesphere.importer.core.service.oauth; + +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.provider.token.ResourceServerTokenServices; + +import edu.asu.diging.citesphere.user.IUser; + +public interface InternalTokenManager extends ResourceServerTokenServices { + + OAuth2AccessToken getAccessToken(IUser user); + +} \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java new file mode 100644 index 0000000..19a32e4 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java @@ -0,0 +1,120 @@ +package edu.asu.diging.citesphere.importer.core.service.oauth.impl; + +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Date; +import java.util.Optional; +import java.util.UUID; + +import javax.annotation.PostConstruct; +import javax.transaction.Transactional; +import javax.transaction.Transactional.TxType; + +import org.javers.common.collections.Sets; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.common.DefaultOAuth2AccessToken; +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.provider.AuthorizationRequest; +import org.springframework.security.oauth2.provider.ClientDetails; +import org.springframework.security.oauth2.provider.ClientDetailsService; +import org.springframework.security.oauth2.provider.OAuth2Authentication; +import org.springframework.security.oauth2.provider.OAuth2Request; +import org.springframework.security.oauth2.provider.OAuth2RequestFactory; +import org.springframework.security.oauth2.provider.TokenRequest; +import org.springframework.security.oauth2.provider.implicit.ImplicitTokenRequest; +import org.springframework.security.oauth2.provider.request.DefaultOAuth2RequestFactory; +import org.springframework.security.oauth2.provider.token.DefaultTokenServices; +import org.springframework.security.oauth2.provider.token.TokenStore; +import org.springframework.stereotype.Service; + +import edu.asu.diging.citesphere.importer.core.service.oauth.OAuthScope; +import edu.asu.diging.citesphere.user.IUser; + +@Service +@PropertySource({ "classpath:config.properties", "${appConfigFile:classpath:}/app.properties" }) +@SuppressWarnings("deprecation") +@Transactional +public class InternalTokenManagerImpl extends DefaultTokenServices implements edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager { + + @Value("${_oauth_token_validity}") + private int tokenValidity; + + @Value("${_citephere_oauth2_app_clientid}") + private String citesphereClientId; + + @Autowired + private TokenStore tokenStore; + + @Autowired + private ClientDetailsService clientDetailsService; + + private OAuth2RequestFactory requestFactory; + + @PostConstruct + public void init() { + this.requestFactory = new DefaultOAuth2RequestFactory(clientDetailsService); + this.setTokenStore(tokenStore); + } + + /* (non-Javadoc) + * @see edu.asu.diging.citesphere.core.service.oauth.impl.InternalTokenManager#getAccessToken() + */ + @Override + @Transactional(TxType.REQUIRES_NEW) + public OAuth2AccessToken getAccessToken(IUser user) { + Collection tokens = tokenStore.findTokensByClientIdAndUserName(citesphereClientId, user.getUsername()); + Optional validToken = tokens.stream().filter(t -> !t.isExpired()).findFirst(); + if (validToken.isPresent()) { + return validToken.get(); + } + return createAccessToken(user); + } + + private OAuth2AccessToken createAccessToken(IUser user) { + DefaultOAuth2AccessToken token = new DefaultOAuth2AccessToken(UUID.randomUUID().toString()); + if (tokenValidity > 0) { + token.setExpiration(new Date(System.currentTimeMillis() + (tokenValidity * 1000L))); + } + token.setScope(Sets.asSet(OAuthScope.READ.getScope())); + + AuthorizationRequest request = new AuthorizationRequest(citesphereClientId, token.getScope()); + TokenRequest implicitRequest = new ImplicitTokenRequest(requestFactory.createTokenRequest(request, "implicit"), requestFactory.createOAuth2Request(request)); + + OAuth2Authentication authentication = getOAuth2Authentication(clientDetailsService.loadClientByClientId(citesphereClientId), implicitRequest, user); + tokenStore.storeAccessToken(token, authentication); + return token; + } + + private OAuth2Authentication getOAuth2Authentication(ClientDetails client, TokenRequest tokenRequest, IUser user) { + OAuth2Request storedOAuth2Request = requestFactory.createOAuth2Request(client, tokenRequest); + Authentication authentication = new UsernamePasswordAuthenticationToken(user, null, user.getRoles()); + return new OAuth2Authentication(storedOAuth2Request, authentication); + } + + private String extractTokenKey(String value) { + if(value == null) { + return null; + } else { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException var5) { + throw new IllegalStateException("MD5 algorithm not available. Fatal (should be in the JDK)."); + } + + try { + byte[] e = digest.digest(value.getBytes("UTF-8")); + return String.format("%032x", new Object[]{new BigInteger(1, e)}); + } catch (UnsupportedEncodingException var4) { + throw new IllegalStateException("UTF-8 encoding not available. Fatal (should be in the JDK)."); + } + } + } +} \ No newline at end of file From 0b3af9b494b73b039b0d911209e329e5e49e57ba Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 30 Jan 2025 12:19:04 -0700 Subject: [PATCH 35/49] [CITE-163] Updated GilesConnector with Giles token from job info --- .../core/service/IGilesConnector.java | 2 +- .../service/giles/impl/GilesConnector.java | 6 ++-- .../impl/CollectionImportProcessor.java | 33 ++++++++----------- .../importer/core/service/impl/JobInfo.java | 7 ++++ 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java index be910cd..320dbd6 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java @@ -7,5 +7,5 @@ public interface IGilesConnector { - IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes); + IGilesUpload uploadFile(IUser user, String token, String filename, byte[] fileBytes); } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java index 9adbb59..a361e6a 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java @@ -44,16 +44,14 @@ public void init() { } @Override - public IGilesUpload uploadFile(IUser user, MultipartFile file, byte[] fileBytes) { - - String token = getToken(user); + public IGilesUpload uploadFile(IUser user, String token, String filename, byte[] fileBytes) { HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); headers.setBearerAuth(token); MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("files", new MultipartFileResource(fileBytes, file.getOriginalFilename())); + body.add("files", new MultipartFileResource(fileBytes, filename)); HttpEntity> requestEntity = new HttpEntity<>(body, headers); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index 822080a..922158e 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -4,6 +4,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -45,6 +47,7 @@ import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; import edu.asu.diging.citesphere.messages.model.ResponseCode; import edu.asu.diging.citesphere.messages.model.Status; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; import edu.asu.diging.citesphere.user.IUser; import edu.asu.diging.simpleusers.core.data.UserRepository; import edu.asu.diging.simpleusers.core.model.impl.User; @@ -165,6 +168,16 @@ public void process(KafkaJobMessage message) { user = (IUser) foundUser.get(); } + File file = new File(root.get(i).get("filePath").asText()); + byte[] fileBytes = null; + try { + fileBytes = Files.readAllBytes(Path.of(root.get(i).get("filePath").asText())); + } catch (IOException e) { + e.printStackTrace(); + } + + IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName() , fileBytes); + } @@ -189,26 +202,6 @@ public void process(KafkaJobMessage message) { } - public static MultipartFile[] getMultipartFilesFromPaths(List filePaths) throws IOException { - List multipartFiles = new ArrayList<>(); - - for (String filePath : filePaths) { - File file = new File(filePath); - - try (FileInputStream inputStream = new FileInputStream(file)) { - MultipartFile multipartFile = new MockMultipartFile( - file.getName(), // Original file name - file.getName(), // File name for upload - "application/octet-stream", // Content type (default to binary) - inputStream // Input stream of the file - ); - multipartFiles.add(multipartFile); - } - } - - return multipartFiles.toArray(new MultipartFile[0]); - } - private void sendMessage(ItemCreationResponse message, String jobId, Status status, ResponseCode code) { KafkaImportReturnMessage returnMessage = new KafkaImportReturnMessage(message, jobId); returnMessage.setStatus(status); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java index 3412f7a..1768227 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/JobInfo.java @@ -9,6 +9,7 @@ public class JobInfo { private String zoteroId; private String groupId; private String collectionId; + private String giles; private String username; public String getZotero() { @@ -35,6 +36,12 @@ public String getCollectionId() { public void setCollectionId(String collectionId) { this.collectionId = collectionId; } + public String getGiles() { + return giles; + } + public void setGiles(String giles) { + this.giles = giles; + } public String getUsername() { return username; } From 908153122896ccff759c07a9830da6aff82bc6bf Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 31 Jan 2025 16:39:00 -0700 Subject: [PATCH 36/49] [CITE-163] Cleaned-up code & fixed some errors --- .../core/service/IGilesConnector.java | 2 - .../service/giles/impl/GilesConnector.java | 19 +-- .../impl/CollectionImportProcessor.java | 51 ++++---- .../service/oauth/InternalTokenManager.java | 12 -- .../oauth/impl/InternalTokenManagerImpl.java | 120 ------------------ .../src/main/resources/config.properties | 3 + 6 files changed, 32 insertions(+), 175 deletions(-) delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java index 320dbd6..a367177 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java @@ -1,7 +1,5 @@ package edu.asu.diging.citesphere.importer.core.service; -import org.springframework.web.multipart.MultipartFile; - import edu.asu.diging.citesphere.model.bib.IGilesUpload; import edu.asu.diging.citesphere.user.IUser; diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java index a361e6a..dbcf940 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java @@ -4,34 +4,29 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ByteArrayResource; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.stereotype.Service; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; -import org.springframework.web.multipart.MultipartFile; import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; -import edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager; import edu.asu.diging.citesphere.model.bib.IGilesUpload; import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; import edu.asu.diging.citesphere.user.IUser; +@Service public class GilesConnector implements IGilesConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); private RestTemplate restTemplate; - - @Autowired - private InternalTokenManager internalTokenManager; - + @Value("${giles_baseurl}") private String gilesBaseurl; @@ -63,10 +58,10 @@ public IGilesUpload uploadFile(IUser user, String token, String filename, byte[] return upload; } - private String getToken(IUser user) { - OAuth2AccessToken token = internalTokenManager.getAccessToken(user); - return token.getValue(); - } +// private String getToken(IUser user) { +// OAuth2AccessToken token = internalTokenManager.getAccessToken(user); +// return token.getValue(); +// } public class MultipartFileResource extends ByteArrayResource { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index 922158e..e63cdf4 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -1,17 +1,19 @@ package edu.asu.diging.citesphere.importer.core.service.impl; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.PostConstruct; @@ -19,7 +21,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -156,45 +157,37 @@ public void process(KafkaJobMessage message) { ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); - for(int i=0; i entry = (Entry) response.getSuccessful().get(i); + + Set> entrySet = response.getSuccessful().entrySet(); + List> entries = new ArrayList(); + entries.addAll((Collection>) entrySet); + + + for (int i = 0; i < entries.size(); i++) { + Map.Entry entry = entries.get(i); + System.out.println("=========================================================="); System.out.println("Key -" + entry.getKey()); - System.out.println("value - "+ entry.getValue()); + System.out.println("Value - " + entry.getValue()); + + String gilesFilePath = root.get(i).get("filePath").asText(); + IUser user = null; - Optional foundUser = userRepository.findById(info.getUsername()); if (foundUser.isPresent()) { - user = (IUser) foundUser.get(); + user = (IUser) foundUser.get(); } - - File file = new File(root.get(i).get("filePath").asText()); + + File file = new File(gilesFilePath); byte[] fileBytes = null; try { - fileBytes = Files.readAllBytes(Path.of(root.get(i).get("filePath").asText())); + fileBytes = Files.readAllBytes(Path.of(gilesFilePath)); } catch (IOException e) { e.printStackTrace(); } - - IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName() , fileBytes); - - + + IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName(), fileBytes); } - -// response.getSuccessful().forEach((key, value) -> { -// System.out.println("=========================================================="); -// System.out.println("Key -" + key.toString()); -// System.out.println("value - "+ value.toString()); -// -// try { -//// Item item = connector.getItem(message.getId(), info.getGroupId(), value.toString().trim()); -//// System.out.println(item.toString()+ " ============================= item"); -// String res = connector.uploadFile(message.getId(), info.getGroupId(), value.toString().trim(), root.get(i).get("filePath")); -// } catch (CitesphereCommunicationException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } -// }); } response = response != null ? response : new ItemCreationResponse(); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java deleted file mode 100644 index ea263b0..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/InternalTokenManager.java +++ /dev/null @@ -1,12 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.oauth; - -import org.springframework.security.oauth2.common.OAuth2AccessToken; -import org.springframework.security.oauth2.provider.token.ResourceServerTokenServices; - -import edu.asu.diging.citesphere.user.IUser; - -public interface InternalTokenManager extends ResourceServerTokenServices { - - OAuth2AccessToken getAccessToken(IUser user); - -} \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java deleted file mode 100644 index 19a32e4..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/impl/InternalTokenManagerImpl.java +++ /dev/null @@ -1,120 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.oauth.impl; - -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Collection; -import java.util.Date; -import java.util.Optional; -import java.util.UUID; - -import javax.annotation.PostConstruct; -import javax.transaction.Transactional; -import javax.transaction.Transactional.TxType; - -import org.javers.common.collections.Sets; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.PropertySource; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.oauth2.common.DefaultOAuth2AccessToken; -import org.springframework.security.oauth2.common.OAuth2AccessToken; -import org.springframework.security.oauth2.provider.AuthorizationRequest; -import org.springframework.security.oauth2.provider.ClientDetails; -import org.springframework.security.oauth2.provider.ClientDetailsService; -import org.springframework.security.oauth2.provider.OAuth2Authentication; -import org.springframework.security.oauth2.provider.OAuth2Request; -import org.springframework.security.oauth2.provider.OAuth2RequestFactory; -import org.springframework.security.oauth2.provider.TokenRequest; -import org.springframework.security.oauth2.provider.implicit.ImplicitTokenRequest; -import org.springframework.security.oauth2.provider.request.DefaultOAuth2RequestFactory; -import org.springframework.security.oauth2.provider.token.DefaultTokenServices; -import org.springframework.security.oauth2.provider.token.TokenStore; -import org.springframework.stereotype.Service; - -import edu.asu.diging.citesphere.importer.core.service.oauth.OAuthScope; -import edu.asu.diging.citesphere.user.IUser; - -@Service -@PropertySource({ "classpath:config.properties", "${appConfigFile:classpath:}/app.properties" }) -@SuppressWarnings("deprecation") -@Transactional -public class InternalTokenManagerImpl extends DefaultTokenServices implements edu.asu.diging.citesphere.importer.core.service.oauth.InternalTokenManager { - - @Value("${_oauth_token_validity}") - private int tokenValidity; - - @Value("${_citephere_oauth2_app_clientid}") - private String citesphereClientId; - - @Autowired - private TokenStore tokenStore; - - @Autowired - private ClientDetailsService clientDetailsService; - - private OAuth2RequestFactory requestFactory; - - @PostConstruct - public void init() { - this.requestFactory = new DefaultOAuth2RequestFactory(clientDetailsService); - this.setTokenStore(tokenStore); - } - - /* (non-Javadoc) - * @see edu.asu.diging.citesphere.core.service.oauth.impl.InternalTokenManager#getAccessToken() - */ - @Override - @Transactional(TxType.REQUIRES_NEW) - public OAuth2AccessToken getAccessToken(IUser user) { - Collection tokens = tokenStore.findTokensByClientIdAndUserName(citesphereClientId, user.getUsername()); - Optional validToken = tokens.stream().filter(t -> !t.isExpired()).findFirst(); - if (validToken.isPresent()) { - return validToken.get(); - } - return createAccessToken(user); - } - - private OAuth2AccessToken createAccessToken(IUser user) { - DefaultOAuth2AccessToken token = new DefaultOAuth2AccessToken(UUID.randomUUID().toString()); - if (tokenValidity > 0) { - token.setExpiration(new Date(System.currentTimeMillis() + (tokenValidity * 1000L))); - } - token.setScope(Sets.asSet(OAuthScope.READ.getScope())); - - AuthorizationRequest request = new AuthorizationRequest(citesphereClientId, token.getScope()); - TokenRequest implicitRequest = new ImplicitTokenRequest(requestFactory.createTokenRequest(request, "implicit"), requestFactory.createOAuth2Request(request)); - - OAuth2Authentication authentication = getOAuth2Authentication(clientDetailsService.loadClientByClientId(citesphereClientId), implicitRequest, user); - tokenStore.storeAccessToken(token, authentication); - return token; - } - - private OAuth2Authentication getOAuth2Authentication(ClientDetails client, TokenRequest tokenRequest, IUser user) { - OAuth2Request storedOAuth2Request = requestFactory.createOAuth2Request(client, tokenRequest); - Authentication authentication = new UsernamePasswordAuthenticationToken(user, null, user.getRoles()); - return new OAuth2Authentication(storedOAuth2Request, authentication); - } - - private String extractTokenKey(String value) { - if(value == null) { - return null; - } else { - MessageDigest digest; - try { - digest = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException var5) { - throw new IllegalStateException("MD5 algorithm not available. Fatal (should be in the JDK)."); - } - - try { - byte[] e = digest.digest(value.getBytes("UTF-8")); - return String.format("%032x", new Object[]{new BigInteger(1, e)}); - } catch (UnsupportedEncodingException var4) { - throw new IllegalStateException("UTF-8 encoding not available. Fatal (should be in the JDK)."); - } - } - } -} \ No newline at end of file diff --git a/citesphere-importer/src/main/resources/config.properties b/citesphere-importer/src/main/resources/config.properties index 543fe4e..db2da3d 100644 --- a/citesphere-importer/src/main/resources/config.properties +++ b/citesphere-importer/src/main/resources/config.properties @@ -26,3 +26,6 @@ _consumer_group=citesphere.importer _zotero_base_url=https://api.zotero.org/ _zotero_template_api_endpoint=/items/new?itemType= _zotero_create_item_api_endpoint=/groups/%s/items + +giles_upload_endpoint=/api/v2/files/upload + From 98319b6c4ba9927b72f3b8f77a64b9434b03c5ce Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 4 Feb 2025 16:34:26 -0700 Subject: [PATCH 37/49] [CITE-163] Updated giles upload part to map files with items --- .../impl/CollectionImportProcessor.java | 114 +++++++++++------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index e63cdf4..5ee686d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -20,6 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.social.zotero.api.Item; import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.JsonProcessingException; @@ -55,7 +56,7 @@ @Service public class CollectionImportProcessor implements IImportProcessor { - + private final Logger logger = LoggerFactory.getLogger(getClass()); @Autowired @@ -69,16 +70,16 @@ public class CollectionImportProcessor implements IImportProcessor { @Autowired private IJsonGenerationService generationService; - + @Autowired private KafkaRequestProducer requestProducer; - + @Autowired private IGilesConnector gilesConnector; - + @Autowired private UserRepository userRepository; - + private Map itemTypeMapping = new HashMap<>(); @PostConstruct @@ -113,7 +114,7 @@ public void process(KafkaJobMessage message) { sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X20); return; } - + sendMessage(null, message.getId(), Status.PROCESSING, ResponseCode.P00); BibEntryIterator bibIterator = null; try { @@ -121,15 +122,16 @@ public void process(KafkaJobMessage message) { } catch (IteratorCreationException e1) { logger.error("Could not create iterator.", e1); } - + if (bibIterator == null) { sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X30); return; } - + ObjectMapper mapper = new ObjectMapper(); ArrayNode root = mapper.createArrayNode(); int entryCounter = 0; + Map filesMap = new HashMap<>(); while (bibIterator.hasNext()) { BibEntry entry = bibIterator.next(); if (entry.getArticleType() == null) { @@ -139,62 +141,35 @@ public void process(KafkaJobMessage message) { ItemType type = itemTypeMapping.get(entry.getArticleType()); JsonNode template = zoteroConnector.getTemplate(type); ObjectNode bibNode = generationService.generateJson(template, entry); - + if(entry.getArticleMeta().getFilePath() != null) { + filesMap.put(entry.getArticleMeta().getArticleTitle(), entry.getArticleMeta().getFilePath()); + } root.add(bibNode); entryCounter++; // we can submit max 50 entries to Zotoro if (entryCounter >= 50) { - submitEntries(root, info); + ItemCreationResponse response = submitEntries(root, info); + addFiles(response, filesMap, message.getId(), info); entryCounter = 0; root = mapper.createArrayNode(); } } - + bibIterator.close(); - + ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); - - Set> entrySet = response.getSuccessful().entrySet(); - List> entries = new ArrayList(); - entries.addAll((Collection>) entrySet); - - - for (int i = 0; i < entries.size(); i++) { - Map.Entry entry = entries.get(i); - - System.out.println("=========================================================="); - System.out.println("Key -" + entry.getKey()); - System.out.println("Value - " + entry.getValue()); - - String gilesFilePath = root.get(i).get("filePath").asText(); - - IUser user = null; - Optional foundUser = userRepository.findById(info.getUsername()); - if (foundUser.isPresent()) { - user = (IUser) foundUser.get(); - } - - File file = new File(gilesFilePath); - byte[] fileBytes = null; - try { - fileBytes = Files.readAllBytes(Path.of(gilesFilePath)); - } catch (IOException e) { - e.printStackTrace(); - } - - IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName(), fileBytes); - } + addFiles(response, filesMap, message.getId(), info); } response = response != null ? response : new ItemCreationResponse(); sendMessage(response, message.getId(), Status.DONE, ResponseCode.S00); } - + private void sendMessage(ItemCreationResponse message, String jobId, Status status, ResponseCode code) { KafkaImportReturnMessage returnMessage = new KafkaImportReturnMessage(message, jobId); returnMessage.setStatus(status); @@ -206,7 +181,7 @@ private void sendMessage(ItemCreationResponse message, String jobId, Status stat logger.error("Exception sending message.", e); } } - + private ItemCreationResponse submitEntries(ArrayNode entries, JobInfo info) { ObjectMapper mapper = new ObjectMapper(); try { @@ -227,7 +202,7 @@ private ItemCreationResponse submitEntries(ArrayNode entries, JobInfo info) { } return null; } - + private JobInfo getJobInfo(KafkaJobMessage message) { JobInfo info = null; try { @@ -252,4 +227,51 @@ private String downloadFile(KafkaJobMessage message) { return file; } + private void addFiles(ItemCreationResponse response, Map filesMap, + String token, JobInfo info) { + + response.getSuccessful().forEach((key, value) -> { + + System.out.println("=========================================================="); + System.out.println("Key -" + key); + System.out.println("Value - " + value); + + + Item item = null; + try { + item = connector.getItem(token, info.getGroupId(), value.toString()); + + + if(filesMap.containsKey(item.getData().getTitle())) { + + String gilesFilePath = filesMap.get(item.getData().getTitle()); + + System.out.println(gilesFilePath + "================================="); + + IUser user = null; + Optional foundUser = userRepository.findById(info.getUsername()); + if (foundUser.isPresent()) { + user = (IUser) foundUser.get(); + } + + File file = new File(gilesFilePath); + byte[] fileBytes = null; + try { + fileBytes = Files.readAllBytes(Path.of(gilesFilePath)); + } catch (IOException e) { + e.printStackTrace(); + } + + IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName(), fileBytes); + +// item.getData().getExtra() + + } + } catch (CitesphereCommunicationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }); + + } } From 26bc2b7ebf69abc8943ed91e6f85ea605738412e Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 6 Feb 2025 12:10:20 -0700 Subject: [PATCH 38/49] [CITE-163] Updated Citesphere cinnecter get Item mapping --- .../core/service/ICitesphereConnector.java | 4 ++-- .../core/service/impl/CitesphereConnector.java | 15 ++++++++++----- .../service/impl/CollectionImportProcessor.java | 14 ++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java index 2f1528d..d291758 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java @@ -1,10 +1,10 @@ package edu.asu.diging.citesphere.importer.core.service; -import org.springframework.social.zotero.api.Item; import org.springframework.web.multipart.MultipartFile; import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; +import edu.asu.diging.citesphere.model.bib.ICitation; public interface ICitesphereConnector { @@ -12,7 +12,7 @@ public interface ICitesphereConnector { String getUploadeFile(String apiToken) throws CitesphereCommunicationException; - Item getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException; + ICitation getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException; String uploadFile(String apiToken, String groupId, String itemKey, MultipartFile[] files) throws CitesphereCommunicationException; } \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java index 6a16829..983aaf7 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java @@ -27,7 +27,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.http.client.ClientHttpRequest; import org.springframework.http.client.ClientHttpResponse; -import org.springframework.social.zotero.api.Item; import org.springframework.stereotype.Service; import org.springframework.web.client.DefaultResponseErrorHandler; import org.springframework.web.client.RequestCallback; @@ -44,6 +43,7 @@ import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.service.CitesphereHeaders; import edu.asu.diging.citesphere.importer.core.service.ICitesphereConnector; +import edu.asu.diging.citesphere.model.bib.ICitation; @Service @PropertySource("classpath:/config.properties") @@ -199,18 +199,23 @@ public void doWithRequest(ClientHttpRequest request) throws IOException { } @Override - public Item getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException { + public ICitation getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException { String path = getItemPath.replace("{groupId}", groupId).replace("{item}", itemKey); @SuppressWarnings("unchecked") ResponseEntity response = (ResponseEntity) makeApiCall(path, apiToken, String.class); HttpStatus status = response.getStatusCode(); - Item item = null; + ICitation citation = null; if (status == HttpStatus.OK) { String responseBody = response.getBody(); ObjectMapper mapper = new ObjectMapper(); try { - item = mapper.readValue(responseBody, Item.class); + JsonNode rootNode = mapper.readTree(responseBody); + JsonNode itemNode = rootNode.get("item"); + + System.out.println(mapper.writeValueAsString(itemNode) + "================================="); + + citation = mapper.readValue(mapper.writeValueAsString(itemNode), ICitation.class); } catch (IOException e) { throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); } @@ -218,7 +223,7 @@ public Item getItem(String apiToken, String groupId, String itemKey) throws Cite throw new CitesphereCommunicationException("Could not communicate with Citesphere properly. Got " + status); } - return item; + return citation; } private ResponseEntity makeApiCall(String url, String apiToken, Class responseType) throws CitesphereCommunicationException { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index 5ee686d..a69df33 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -5,22 +5,15 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.social.zotero.api.Item; import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.JsonProcessingException; @@ -49,6 +42,7 @@ import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; import edu.asu.diging.citesphere.messages.model.ResponseCode; import edu.asu.diging.citesphere.messages.model.Status; +import edu.asu.diging.citesphere.model.bib.ICitation; import edu.asu.diging.citesphere.model.bib.IGilesUpload; import edu.asu.diging.citesphere.user.IUser; import edu.asu.diging.simpleusers.core.data.UserRepository; @@ -237,14 +231,14 @@ private void addFiles(ItemCreationResponse response, Map filesMa System.out.println("Value - " + value); - Item item = null; + ICitation item = null; try { item = connector.getItem(token, info.getGroupId(), value.toString()); - if(filesMap.containsKey(item.getData().getTitle())) { + if(filesMap.containsKey(item.getTitle())) { - String gilesFilePath = filesMap.get(item.getData().getTitle()); + String gilesFilePath = filesMap.get(item.getTitle()); System.out.println(gilesFilePath + "================================="); From 91c138064f1158b28c44f44ca1e56947b1b9cf98 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 7 Feb 2025 16:52:20 -0700 Subject: [PATCH 39/49] [CITE-163] Updated BibEntry and ExtraFieldHelper for giles upload --- .../importer/core/model/impl/ArticleMeta.java | 9 +++ .../impl/CollectionImportProcessor.java | 57 +------------------ .../parse/iterators/BibFileIterator.java | 40 ++++++++++++- .../zotero/template/ItemJsonGenerator.java | 1 + .../template/impl/ExtraFieldHelper.java | 10 ++++ 5 files changed, 61 insertions(+), 56 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java index 38432f3..9c354b1 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/model/impl/ArticleMeta.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; + public class ArticleMeta { private List collections; @@ -50,6 +52,7 @@ public class ArticleMeta { private List references; private String referenceCount; private String retrievalDate; + private List gilesUpload; public List getCollections() { return collections; @@ -303,5 +306,11 @@ public String getRetrievalDate() { public void setRetrievalDate(String retrievalDate) { this.retrievalDate = retrievalDate; } + public List getGilesUpload() { + return gilesUpload; + } + public void setGilesUpload(List gilesUpload) { + this.gilesUpload = gilesUpload; + } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index a69df33..bf15595 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -68,12 +68,6 @@ public class CollectionImportProcessor implements IImportProcessor { @Autowired private KafkaRequestProducer requestProducer; - @Autowired - private IGilesConnector gilesConnector; - - @Autowired - private UserRepository userRepository; - private Map itemTypeMapping = new HashMap<>(); @PostConstruct @@ -144,7 +138,7 @@ public void process(KafkaJobMessage message) { // we can submit max 50 entries to Zotoro if (entryCounter >= 50) { ItemCreationResponse response = submitEntries(root, info); - addFiles(response, filesMap, message.getId(), info); +// addFiles(response, filesMap, message.getId(), info); entryCounter = 0; root = mapper.createArrayNode(); } @@ -156,7 +150,7 @@ public void process(KafkaJobMessage message) { ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); - addFiles(response, filesMap, message.getId(), info); +// addFiles(response, filesMap, message.getId(), info); } response = response != null ? response : new ItemCreationResponse(); @@ -221,51 +215,4 @@ private String downloadFile(KafkaJobMessage message) { return file; } - private void addFiles(ItemCreationResponse response, Map filesMap, - String token, JobInfo info) { - - response.getSuccessful().forEach((key, value) -> { - - System.out.println("=========================================================="); - System.out.println("Key -" + key); - System.out.println("Value - " + value); - - - ICitation item = null; - try { - item = connector.getItem(token, info.getGroupId(), value.toString()); - - - if(filesMap.containsKey(item.getTitle())) { - - String gilesFilePath = filesMap.get(item.getTitle()); - - System.out.println(gilesFilePath + "================================="); - - IUser user = null; - Optional foundUser = userRepository.findById(info.getUsername()); - if (foundUser.isPresent()) { - user = (IUser) foundUser.get(); - } - - File file = new File(gilesFilePath); - byte[] fileBytes = null; - try { - fileBytes = Files.readAllBytes(Path.of(gilesFilePath)); - } catch (IOException e) { - e.printStackTrace(); - } - - IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName(), fileBytes); - -// item.getData().getExtra() - - } - } catch (CitesphereCommunicationException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - }); - - } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index c500817..971d32a 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -2,13 +2,15 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.commons.io.FileUtils; @@ -18,6 +20,7 @@ import org.springframework.social.zotero.api.Item; import edu.asu.diging.citesphere.factory.impl.ParseExtra; +import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleId; @@ -29,14 +32,22 @@ import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.model.impl.Reference; +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; +import edu.asu.diging.citesphere.importer.core.service.giles.impl.GilesConnector; +import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.model.bib.IAffiliation; import edu.asu.diging.citesphere.model.bib.ICitation; import edu.asu.diging.citesphere.model.bib.ICreator; +import edu.asu.diging.citesphere.model.bib.IGilesUpload; import edu.asu.diging.citesphere.model.bib.IPerson; import edu.asu.diging.citesphere.model.bib.IReference; import edu.asu.diging.citesphere.model.bib.impl.Citation; +import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; import edu.asu.diging.citesphere.model.bib.impl.Person; +import edu.asu.diging.citesphere.user.IUser; +import edu.asu.diging.simpleusers.core.data.UserRepository; +import edu.asu.diging.simpleusers.core.model.impl.User; public class BibFileIterator implements BibEntryIterator { @@ -49,11 +60,15 @@ public class BibFileIterator implements BibEntryIterator { private String collectionId; private Iterator lineIterator; private Map typeMap; + private IGilesConnector gilesConnector; + private UserRepository userRepository; public BibFileIterator(String filePath, String groupId, String collectionId) { this.filePath = filePath; this.groupId = groupId; this.collectionId = collectionId; + gilesConnector = new GilesConnector(); + userRepository = new UserRepository(); parseExtra = new ParseExtra(); parseExtra.init(); init(); @@ -233,6 +248,8 @@ private ArticleMeta parseArticleMeta(Map fields) { meta.setDocumentType(fileParts[2]); meta.setFilePath(fileParts[1]); } + + return meta; } @@ -305,6 +322,27 @@ private Reference mapSingleReference(IReference citationRef) { return ref; } + private IGilesUpload createGilesUpload(String gilesFilePath, JobInfo info) { + System.out.println(gilesFilePath + "================================="); + + IUser user = null; + Optional foundUser = userRepository.findById(info.getUsername()); + if (foundUser.isPresent()) { + user = (IUser) foundUser.get(); + } + + File file = new File(gilesFilePath); + byte[] fileBytes = null; + try { + fileBytes = Files.readAllBytes(Path.of(gilesFilePath)); + } catch (IOException e) { + e.printStackTrace(); + } + + IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName(), fileBytes); + return upload; + } + @Override public boolean hasNext() { return lineIterator.hasNext(); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java index 5915da0..d594192 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/ItemJsonGenerator.java @@ -283,6 +283,7 @@ public String processExtra(JsonNode node, BibEntry article) { fieldHelper.createReferences(article, root); fieldHelper.createReferenceCount(article, root); fieldHelper.createRetrievalDate(article, root); + fieldHelper.createGilesUpload(article, root); try { return prefix + getObjectMapper().writeValueAsString(root); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java index 5247b07..83fa52a 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java @@ -9,6 +9,8 @@ import org.springframework.stereotype.Component; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -424,6 +426,14 @@ public void createRetrievalDate(BibEntry article, ObjectNode root) { root.put("retrievalDate", article.getArticleMeta().getRetrievalDate()); } } + + public void createGilesUpload(BibEntry article, ObjectNode root) { + if (article.getArticleMeta().getGilesUpload() != null) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode gilesJson = objectMapper.valueToTree(article.getArticleMeta().getGilesUpload()); + root.put("gilesUploads", gilesJson); + } + } private void fillPerson(Contributor contrib, ObjectNode creatorNode, int idx) { List nameParts = new ArrayList<>(); From 7d4c3327a545ad96de44c0fc94a054a08dfca4fc Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 11 Feb 2025 16:40:19 -0700 Subject: [PATCH 40/49] [CITE-163] Updated bibFileIterator & tested the giles upload --- .../service/parse/impl/BibFileHandler.java | 7 ++++++- .../parse/iterators/BibFileIterator.java | 21 ++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index 8abe1d1..feb1593 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -1,5 +1,6 @@ package edu.asu.diging.citesphere.importer.core.service.parse.impl; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; @@ -9,10 +10,14 @@ import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; import edu.asu.diging.citesphere.importer.core.service.parse.iterators.BibFileIterator; +import edu.asu.diging.simpleusers.core.data.UserRepository; @Service public class BibFileHandler implements FileHandler { + @Autowired + private UserRepository userRepository; + @Override public boolean canHandle(String path) throws HandlerTestException { if (path.toLowerCase().endsWith(".bib")) { @@ -24,7 +29,7 @@ public boolean canHandle(String path) throws HandlerTestException { @Override public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) throws IteratorCreationException { - return new BibFileIterator(path, info.getGroupId(), info.getCollectionId()); + return new BibFileIterator(path, info, userRepository); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 971d32a..72f606b 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -58,17 +58,19 @@ public class BibFileIterator implements BibEntryIterator { private String filePath; private String groupId; private String collectionId; + private JobInfo info; private Iterator lineIterator; private Map typeMap; private IGilesConnector gilesConnector; private UserRepository userRepository; - public BibFileIterator(String filePath, String groupId, String collectionId) { + public BibFileIterator(String filePath, JobInfo info, UserRepository userRepository) { this.filePath = filePath; - this.groupId = groupId; - this.collectionId = collectionId; - gilesConnector = new GilesConnector(); - userRepository = new UserRepository(); + this.groupId = info.getGroupId(); + this.collectionId = info.getCollectionId(); + this.info = info; + this.gilesConnector = new GilesConnector(); + this.userRepository = userRepository; parseExtra = new ParseExtra(); parseExtra.init(); init(); @@ -245,8 +247,13 @@ private ArticleMeta parseArticleMeta(Map fields) { if(fields.containsKey("file")) { String[] fileParts = fields.get("file").split(":"); - meta.setDocumentType(fileParts[2]); - meta.setFilePath(fileParts[1]); +// meta.setDocumentType(fileParts[2]); +// meta.setFilePath(fileParts[1]); + IGilesUpload upload = createGilesUpload(fileParts[1], info); + List uploads = new ArrayList<>(); + uploads.add(upload); + meta.setGilesUpload(uploads); +// ================= } From f73265265d84bbedd9299b75a9f1879e3a35c3bb Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Tue, 18 Feb 2025 16:38:39 -0700 Subject: [PATCH 41/49] [CITE-163] Address the User class error --- .../importer/core/repository/UserRepository.java | 16 ++++++++++++++++ .../service/impl/CollectionImportProcessor.java | 11 ----------- .../core/service/parse/impl/BibFileHandler.java | 3 ++- .../service/parse/iterators/BibFileIterator.java | 6 ++---- 4 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java new file mode 100644 index 0000000..6c20403 --- /dev/null +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java @@ -0,0 +1,16 @@ +package edu.asu.diging.citesphere.importer.core.repository; + +import java.util.List; + +import org.javers.spring.annotation.JaversSpringDataAuditable; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.stereotype.Repository; + +import edu.asu.diging.citesphere.user.impl.User; + +//@JaversSpringDataAuditable +//@Repository +public interface UserRepository extends PagingAndSortingRepository { + + public List findByEmail(String email); +} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index bf15595..d937a64 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -1,13 +1,8 @@ package edu.asu.diging.citesphere.importer.core.service.impl; -import java.io.File; -import java.io.IOException; import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import javax.annotation.PostConstruct; @@ -30,7 +25,6 @@ import edu.asu.diging.citesphere.importer.core.model.ItemType; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.service.ICitesphereConnector; -import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; import edu.asu.diging.citesphere.importer.core.service.IImportProcessor; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; @@ -42,11 +36,6 @@ import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; import edu.asu.diging.citesphere.messages.model.ResponseCode; import edu.asu.diging.citesphere.messages.model.Status; -import edu.asu.diging.citesphere.model.bib.ICitation; -import edu.asu.diging.citesphere.model.bib.IGilesUpload; -import edu.asu.diging.citesphere.user.IUser; -import edu.asu.diging.simpleusers.core.data.UserRepository; -import edu.asu.diging.simpleusers.core.model.impl.User; @Service public class CollectionImportProcessor implements IImportProcessor { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index feb1593..9d4f0c3 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -5,12 +5,13 @@ import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; +import edu.asu.diging.citesphere.importer.core.repository.UserRepository; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; import edu.asu.diging.citesphere.importer.core.service.parse.iterators.BibFileIterator; -import edu.asu.diging.simpleusers.core.data.UserRepository; + @Service public class BibFileHandler implements FileHandler { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 72f606b..e239d89 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -20,7 +20,6 @@ import org.springframework.social.zotero.api.Item; import edu.asu.diging.citesphere.factory.impl.ParseExtra; -import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleId; @@ -32,6 +31,7 @@ import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.model.impl.Reference; +import edu.asu.diging.citesphere.importer.core.repository.UserRepository; import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; import edu.asu.diging.citesphere.importer.core.service.giles.impl.GilesConnector; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; @@ -43,11 +43,9 @@ import edu.asu.diging.citesphere.model.bib.IPerson; import edu.asu.diging.citesphere.model.bib.IReference; import edu.asu.diging.citesphere.model.bib.impl.Citation; -import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; import edu.asu.diging.citesphere.model.bib.impl.Person; import edu.asu.diging.citesphere.user.IUser; -import edu.asu.diging.simpleusers.core.data.UserRepository; -import edu.asu.diging.simpleusers.core.model.impl.User; +import edu.asu.diging.citesphere.user.impl.User; public class BibFileIterator implements BibEntryIterator { From fd79011255291831ec00ce61eaaddcba3dff6828 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 20 Feb 2025 12:15:34 -0700 Subject: [PATCH 42/49] [CITE-163] Fixed user issue in giles connector --- .../core/repository/UserRepository.java | 16 ---------- .../core/service/IGilesConnector.java | 3 +- .../service/giles/impl/GilesConnector.java | 30 ++++++++++++++++--- .../service/parse/impl/BibFileHandler.java | 7 +---- .../parse/iterators/BibFileIterator.java | 23 +++++--------- 5 files changed, 36 insertions(+), 43 deletions(-) delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java deleted file mode 100644 index 6c20403..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/repository/UserRepository.java +++ /dev/null @@ -1,16 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.repository; - -import java.util.List; - -import org.javers.spring.annotation.JaversSpringDataAuditable; -import org.springframework.data.repository.PagingAndSortingRepository; -import org.springframework.stereotype.Repository; - -import edu.asu.diging.citesphere.user.impl.User; - -//@JaversSpringDataAuditable -//@Repository -public interface UserRepository extends PagingAndSortingRepository { - - public List findByEmail(String email); -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java index a367177..a7f7250 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/IGilesConnector.java @@ -1,9 +1,8 @@ package edu.asu.diging.citesphere.importer.core.service; import edu.asu.diging.citesphere.model.bib.IGilesUpload; -import edu.asu.diging.citesphere.user.IUser; public interface IGilesConnector { - IGilesUpload uploadFile(IUser user, String token, String filename, byte[] fileBytes); + IGilesUpload uploadFile(String username, String token, String filename, byte[] fileBytes); } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java index dbcf940..0b20935 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java @@ -4,10 +4,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; import org.springframework.core.io.ByteArrayResource; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; @@ -18,9 +21,10 @@ import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; import edu.asu.diging.citesphere.model.bib.IGilesUpload; import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; -import edu.asu.diging.citesphere.user.IUser; @Service +@PropertySource({ "classpath:/config.properties", + "${appConfigFile:classpath:}/app.properties" }) public class GilesConnector implements IGilesConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -36,10 +40,18 @@ public class GilesConnector implements IGilesConnector { @PostConstruct public void init() { restTemplate = new RestTemplate(); + logger.info("GilesConnector initialized with: ================================================"); + logger.info("Giles Base URL: " + gilesBaseurl); + logger.info("Upload Endpoint: " + uploadEndpoint); + logger.info("RestTemplate: " + (restTemplate != null ? "Injected" : "NULL")); } @Override - public IGilesUpload uploadFile(IUser user, String token, String filename, byte[] fileBytes) { + public IGilesUpload uploadFile(String username, String token, String filename, byte[] fileBytes) { + + if (restTemplate == null) { + restTemplate = new RestTemplate(); // Ensure restTemplate is initialized + } HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); @@ -50,11 +62,21 @@ public IGilesUpload uploadFile(IUser user, String token, String filename, byte[] HttpEntity> requestEntity = new HttpEntity<>(body, headers); - ResponseEntity response = restTemplate.postForEntity(gilesBaseurl + uploadEndpoint, requestEntity, UploadResponse.class); + ResponseEntity response = null; + try { + System.out.println(gilesBaseurl + " " + uploadEndpoint ); + System.out.println(restTemplate.toString()); + logger.info("Uploading file to: ================================= " + gilesBaseurl + uploadEndpoint); + response = restTemplate.exchange(gilesBaseurl + uploadEndpoint, + HttpMethod.POST, requestEntity, UploadResponse.class); + } catch (Exception ex) { + logger.error("Upload request failed", ex); + return null; // Handle failure gracefully + } IGilesUpload upload = new GilesUpload(); upload.setProgressId(response.getBody().getId()); - upload.setUploadingUser(user.getUsername()); + upload.setUploadingUser(username); return upload; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index 9d4f0c3..9f548b0 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -1,11 +1,9 @@ package edu.asu.diging.citesphere.importer.core.service.parse.impl; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; -import edu.asu.diging.citesphere.importer.core.repository.UserRepository; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; @@ -16,9 +14,6 @@ @Service public class BibFileHandler implements FileHandler { - @Autowired - private UserRepository userRepository; - @Override public boolean canHandle(String path) throws HandlerTestException { if (path.toLowerCase().endsWith(".bib")) { @@ -30,7 +25,7 @@ public boolean canHandle(String path) throws HandlerTestException { @Override public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) throws IteratorCreationException { - return new BibFileIterator(path, info, userRepository); + return new BibFileIterator(path, info); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index e239d89..c7c757c 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -4,13 +4,13 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import org.apache.commons.io.FileUtils; @@ -31,7 +31,6 @@ import edu.asu.diging.citesphere.importer.core.model.impl.Issn; import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.model.impl.Reference; -import edu.asu.diging.citesphere.importer.core.repository.UserRepository; import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; import edu.asu.diging.citesphere.importer.core.service.giles.impl.GilesConnector; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; @@ -44,8 +43,6 @@ import edu.asu.diging.citesphere.model.bib.IReference; import edu.asu.diging.citesphere.model.bib.impl.Citation; import edu.asu.diging.citesphere.model.bib.impl.Person; -import edu.asu.diging.citesphere.user.IUser; -import edu.asu.diging.citesphere.user.impl.User; public class BibFileIterator implements BibEntryIterator { @@ -60,15 +57,13 @@ public class BibFileIterator implements BibEntryIterator { private Iterator lineIterator; private Map typeMap; private IGilesConnector gilesConnector; - private UserRepository userRepository; - public BibFileIterator(String filePath, JobInfo info, UserRepository userRepository) { + public BibFileIterator(String filePath, JobInfo info) { this.filePath = filePath; this.groupId = info.getGroupId(); this.collectionId = info.getCollectionId(); this.info = info; this.gilesConnector = new GilesConnector(); - this.userRepository = userRepository; parseExtra = new ParseExtra(); parseExtra.init(); init(); @@ -76,6 +71,7 @@ public BibFileIterator(String filePath, JobInfo info, UserRepository userReposit private void init() { try { + System.out.println("filePath ============================ " + filePath); lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); } catch (IOException e) { logger.error("Could not create line iterator.", e); @@ -244,10 +240,13 @@ private ArticleMeta parseArticleMeta(Map fields) { } if(fields.containsKey("file")) { + Path path = Paths.get(filePath); + Path folderPath = path.getParent(); String[] fileParts = fields.get("file").split(":"); // meta.setDocumentType(fileParts[2]); // meta.setFilePath(fileParts[1]); - IGilesUpload upload = createGilesUpload(fileParts[1], info); + System.out.println(folderPath.toString()+"/"+fileParts[1]); + IGilesUpload upload = createGilesUpload(folderPath.toString()+"/"+fileParts[1], info); List uploads = new ArrayList<>(); uploads.add(upload); meta.setGilesUpload(uploads); @@ -330,12 +329,6 @@ private Reference mapSingleReference(IReference citationRef) { private IGilesUpload createGilesUpload(String gilesFilePath, JobInfo info) { System.out.println(gilesFilePath + "================================="); - IUser user = null; - Optional foundUser = userRepository.findById(info.getUsername()); - if (foundUser.isPresent()) { - user = (IUser) foundUser.get(); - } - File file = new File(gilesFilePath); byte[] fileBytes = null; try { @@ -344,7 +337,7 @@ private IGilesUpload createGilesUpload(String gilesFilePath, JobInfo info) { e.printStackTrace(); } - IGilesUpload upload = gilesConnector.uploadFile(user, info.getGiles(), file.getName(), fileBytes); + IGilesUpload upload = gilesConnector.uploadFile(info.getUsername(), info.getGiles(), file.getName(), fileBytes); return upload; } From b8905520510213e1ab7071f243ca45318131e670 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 21 Feb 2025 16:38:04 -0700 Subject: [PATCH 43/49] [CITE-163] Resolved the null pointer exception related to GilesConnector --- .../importer/core/service/parse/impl/BibFileHandler.java | 7 ++++++- .../core/service/parse/iterators/BibFileIterator.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java index 9f548b0..d5c998d 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/impl/BibFileHandler.java @@ -1,9 +1,11 @@ package edu.asu.diging.citesphere.importer.core.service.parse.impl; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import edu.asu.diging.citesphere.importer.core.exception.HandlerTestException; import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; +import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.importer.core.service.parse.FileHandler; @@ -14,6 +16,9 @@ @Service public class BibFileHandler implements FileHandler { + @Autowired + private IGilesConnector gilesConnector; + @Override public boolean canHandle(String path) throws HandlerTestException { if (path.toLowerCase().endsWith(".bib")) { @@ -25,7 +30,7 @@ public boolean canHandle(String path) throws HandlerTestException { @Override public BibEntryIterator getIterator(String path, IHandlerRegistry callback, JobInfo info) throws IteratorCreationException { - return new BibFileIterator(path, info); + return new BibFileIterator(path, info, gilesConnector); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index c7c757c..f3830b4 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -58,12 +58,12 @@ public class BibFileIterator implements BibEntryIterator { private Map typeMap; private IGilesConnector gilesConnector; - public BibFileIterator(String filePath, JobInfo info) { + public BibFileIterator(String filePath, JobInfo info, IGilesConnector gilesConnector) { this.filePath = filePath; this.groupId = info.getGroupId(); this.collectionId = info.getCollectionId(); this.info = info; - this.gilesConnector = new GilesConnector(); + this.gilesConnector = gilesConnector; parseExtra = new ParseExtra(); parseExtra.init(); init(); From 725c08bdb097ec80b973e61457cd1701925f2064 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 27 Feb 2025 11:01:16 -0700 Subject: [PATCH 44/49] [CITEE-163] Cleaned up code --- citesphere-importer/pom.xml | 10 +- .../core/service/ICitesphereConnector.java | 6 -- .../service/giles/impl/GilesConnector.java | 26 +---- .../service/impl/CitesphereConnector.java | 94 +------------------ .../impl/CollectionImportProcessor.java | 2 - .../core/service/oauth/OAuthScope.java | 16 ---- .../parse/iterators/BibFileIterator.java | 6 -- .../parse/jstor/xml/BibEntryParser.java | 24 ----- .../parse/jstor/xml/IBibEntryParser.java | 14 --- .../src/main/resources/app.properties | 17 ---- .../src/main/resources/config.properties | 3 +- 11 files changed, 19 insertions(+), 199 deletions(-) delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java delete mode 100644 citesphere-importer/src/main/resources/app.properties diff --git a/citesphere-importer/pom.xml b/citesphere-importer/pom.xml index bb6727d..ad306b9 100644 --- a/citesphere-importer/pom.xml +++ b/citesphere-importer/pom.xml @@ -15,8 +15,8 @@ Lovelace-RELEASE 3.0.11.RELEASE 2.2.6.RELEASE - 0.5 - 1.22 + 0.6 + 1.24 2.4.0.RELEASE $2a$04$oQo44vqcDIFRoYKiAXoNheurzkwX9dcNmowvTX/hsWuBMwijqn44i @@ -314,6 +314,12 @@ 1.10.19 test + + + org.springframework + spring-test + test + com.opencsv diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java index d291758..dc81f68 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java @@ -1,10 +1,7 @@ package edu.asu.diging.citesphere.importer.core.service; -import org.springframework.web.multipart.MultipartFile; - import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; -import edu.asu.diging.citesphere.model.bib.ICitation; public interface ICitesphereConnector { @@ -12,7 +9,4 @@ public interface ICitesphereConnector { String getUploadeFile(String apiToken) throws CitesphereCommunicationException; - ICitation getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException; - - String uploadFile(String apiToken, String groupId, String itemKey, MultipartFile[] files) throws CitesphereCommunicationException; } \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java index 0b20935..efec994 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/giles/impl/GilesConnector.java @@ -4,7 +4,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; import org.springframework.core.io.ByteArrayResource; @@ -23,27 +22,22 @@ import edu.asu.diging.citesphere.model.bib.impl.GilesUpload; @Service -@PropertySource({ "classpath:/config.properties", - "${appConfigFile:classpath:}/app.properties" }) +@PropertySource("classpath:/config.properties") public class GilesConnector implements IGilesConnector { private final Logger logger = LoggerFactory.getLogger(getClass()); private RestTemplate restTemplate; - @Value("${giles_baseurl}") + @Value("${_giles_baseurl}") private String gilesBaseurl; - @Value("${giles_upload_endpoint}") + @Value("${_giles_upload_endpoint}") private String uploadEndpoint; @PostConstruct public void init() { restTemplate = new RestTemplate(); - logger.info("GilesConnector initialized with: ================================================"); - logger.info("Giles Base URL: " + gilesBaseurl); - logger.info("Upload Endpoint: " + uploadEndpoint); - logger.info("RestTemplate: " + (restTemplate != null ? "Injected" : "NULL")); } @Override @@ -64,15 +58,10 @@ public IGilesUpload uploadFile(String username, String token, String filename, b ResponseEntity response = null; try { - System.out.println(gilesBaseurl + " " + uploadEndpoint ); - System.out.println(restTemplate.toString()); - logger.info("Uploading file to: ================================= " + gilesBaseurl + uploadEndpoint); - response = restTemplate.exchange(gilesBaseurl + uploadEndpoint, - HttpMethod.POST, requestEntity, UploadResponse.class); - + response = restTemplate.exchange(gilesBaseurl + uploadEndpoint, HttpMethod.POST, requestEntity, UploadResponse.class); } catch (Exception ex) { logger.error("Upload request failed", ex); - return null; // Handle failure gracefully + return null; } IGilesUpload upload = new GilesUpload(); upload.setProgressId(response.getBody().getId()); @@ -80,11 +69,6 @@ public IGilesUpload uploadFile(String username, String token, String filename, b return upload; } -// private String getToken(IUser user) { -// OAuth2AccessToken token = internalTokenManager.getAccessToken(user); -// return token.getValue(); -// } - public class MultipartFileResource extends ByteArrayResource { private String filename; diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java index 983aaf7..8efd2d3 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java @@ -34,7 +34,6 @@ import org.springframework.web.client.ResponseExtractor; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; -import org.springframework.web.multipart.MultipartFile; import org.springframework.web.util.DefaultUriBuilderFactory; import com.fasterxml.jackson.databind.JsonNode; @@ -43,7 +42,6 @@ import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; import edu.asu.diging.citesphere.importer.core.service.CitesphereHeaders; import edu.asu.diging.citesphere.importer.core.service.ICitesphereConnector; -import edu.asu.diging.citesphere.model.bib.ICitation; @Service @PropertySource("classpath:/config.properties") @@ -72,12 +70,6 @@ public class CitesphereConnector implements ICitesphereConnector { @Value("${_citesphere_job_info_path}") private String jobInfoPath; - @Value("${_citesphere_get_item_path}") - private String getItemPath; - - @Value("${_citesphere_upload_file_path}") - private String uploadFilePath; - private RestTemplate restTemplate; private String accessToken; @@ -198,36 +190,8 @@ public void doWithRequest(ClientHttpRequest request) throws IOException { return filepath; } - @Override - public ICitation getItem(String apiToken, String groupId, String itemKey) throws CitesphereCommunicationException { - String path = getItemPath.replace("{groupId}", groupId).replace("{item}", itemKey); - @SuppressWarnings("unchecked") - ResponseEntity response = (ResponseEntity) makeApiCall(path, apiToken, String.class); - HttpStatus status = response.getStatusCode(); - - ICitation citation = null; - if (status == HttpStatus.OK) { - String responseBody = response.getBody(); - ObjectMapper mapper = new ObjectMapper(); - try { - JsonNode rootNode = mapper.readTree(responseBody); - JsonNode itemNode = rootNode.get("item"); - - System.out.println(mapper.writeValueAsString(itemNode) + "================================="); - - citation = mapper.readValue(mapper.writeValueAsString(itemNode), ICitation.class); - } catch (IOException e) { - throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); - } - } else { - throw new CitesphereCommunicationException("Could not communicate with Citesphere properly. Got " + status); - } - - return citation; - } - private ResponseEntity makeApiCall(String url, String apiToken, Class responseType) throws CitesphereCommunicationException { - HttpEntity entity = buildHeaders(apiToken, "body"); + HttpEntity entity = buildHeaders(apiToken); ResponseEntity response; try { response = restTemplate.exchange(url, HttpMethod.GET, entity, responseType); @@ -243,36 +207,13 @@ private ResponseEntity makeApiCall(String url, String apiToken, Class resp } // let's try again after getting a new OAuth token - entity = buildHeaders(apiToken, "body"); - response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class); - } - return response; - } - - private ResponseEntity makePostApiCall(String url, String apiToken, Object requestBody, Class responseType) throws CitesphereCommunicationException { - HttpEntity entity = buildHeaders(apiToken, requestBody); - ResponseEntity response; - try { - response = restTemplate.exchange(url, HttpMethod.POST, entity, responseType); - } catch (RestClientException ex) { - throw new CitesphereCommunicationException("Could not understand server.", ex); - } - HttpStatus status = response.getStatusCode(); - - if (status == HttpStatus.UNAUTHORIZED) { - String responseBody = response.getBody().toString(); - if (!refreshToken(responseBody)) { - throw new CitesphereCommunicationException("Could not understand returned error message: " + responseBody); - } - - // let's try again after getting a new OAuth token - entity = buildHeaders(apiToken, "body"); + entity = buildHeaders(apiToken); response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class); } return response; } - private HttpEntity buildHeaders(String apiToken, Object requestBody) { + private HttpEntity buildHeaders(String apiToken) { if (accessToken == null) { accessToken = getAccessToken(); } @@ -280,7 +221,7 @@ private HttpEntity buildHeaders(String apiToken, Object requestBody) { HttpHeaders headers = new HttpHeaders(); headers.setBearerAuth(accessToken); headers.put(CitesphereHeaders.CITESPHERE_API_TOKEN, Arrays.asList(apiToken)); - HttpEntity entity = new HttpEntity<>(requestBody, headers); + HttpEntity entity = new HttpEntity<>("body", headers); return entity; } @@ -322,31 +263,4 @@ private boolean refreshToken(String errorMessage) { return false; } - @Override - public String uploadFile(String apiToken, String groupId, String itemKey, MultipartFile[] files) throws CitesphereCommunicationException { - String path = uploadFilePath.replace("{groupId}", groupId).replace("{item}", itemKey); - - Map requestBody = new HashMap<>(); - requestBody.put("files", files); - - @SuppressWarnings("unchecked") - ResponseEntity response = (ResponseEntity) makePostApiCall(path, apiToken, requestBody, String.class); - HttpStatus status = response.getStatusCode(); - -// Item item = null; - if (status == HttpStatus.OK) { - String responseBody = response.getBody(); - System.out.println("=========================== response - " + responseBody); - ObjectMapper mapper = new ObjectMapper(); -// try { -// item = mapper.readValue(responseBody, Item.class); -// } catch (IOException e) { -// throw new CitesphereCommunicationException("Could not understand returned message: " + responseBody, e); -// } - } else { - throw new CitesphereCommunicationException("Could not communicate with Citesphere properly. Got " + status); - } - - return response.getBody(); - } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index d937a64..f431c3f 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -127,7 +127,6 @@ public void process(KafkaJobMessage message) { // we can submit max 50 entries to Zotoro if (entryCounter >= 50) { ItemCreationResponse response = submitEntries(root, info); -// addFiles(response, filesMap, message.getId(), info); entryCounter = 0; root = mapper.createArrayNode(); } @@ -139,7 +138,6 @@ public void process(KafkaJobMessage message) { ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); -// addFiles(response, filesMap, message.getId(), info); } response = response != null ? response : new ItemCreationResponse(); diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java deleted file mode 100644 index db8a49c..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/oauth/OAuthScope.java +++ /dev/null @@ -1,16 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.oauth; - -public enum OAuthScope { - READ("read"), - WRITE("write"); - - private String scope; - - private OAuthScope(String scope) { - this.scope = scope; - } - - public String getScope() { - return scope; - } -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index f3830b4..d8f8d77 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -71,7 +71,6 @@ public BibFileIterator(String filePath, JobInfo info, IGilesConnector gilesConne private void init() { try { - System.out.println("filePath ============================ " + filePath); lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); } catch (IOException e) { logger.error("Could not create line iterator.", e); @@ -243,14 +242,11 @@ private ArticleMeta parseArticleMeta(Map fields) { Path path = Paths.get(filePath); Path folderPath = path.getParent(); String[] fileParts = fields.get("file").split(":"); -// meta.setDocumentType(fileParts[2]); -// meta.setFilePath(fileParts[1]); System.out.println(folderPath.toString()+"/"+fileParts[1]); IGilesUpload upload = createGilesUpload(folderPath.toString()+"/"+fileParts[1], info); List uploads = new ArrayList<>(); uploads.add(upload); meta.setGilesUpload(uploads); -// ================= } @@ -327,8 +323,6 @@ private Reference mapSingleReference(IReference citationRef) { } private IGilesUpload createGilesUpload(String gilesFilePath, JobInfo info) { - System.out.println(gilesFilePath + "================================="); - File file = new File(gilesFilePath); byte[] fileBytes = null; try { diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java deleted file mode 100644 index 9c02fd2..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/BibEntryParser.java +++ /dev/null @@ -1,24 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml; - -import org.springframework.stereotype.Component; -import org.w3c.dom.Node; - -import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; -import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; - -@Component -public class BibEntryParser implements IBibEntryParser { - - @Override - public void parseJournalMetaTag(Node node, ContainerMeta meta) { - // TODO Auto-generated method stub - - } - - @Override - public void parseArticleMetaTag(Node node, ArticleMeta meta) { - // TODO Auto-generated method stub - - } - -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java deleted file mode 100644 index 6e0878a..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/jstor/xml/IBibEntryParser.java +++ /dev/null @@ -1,14 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.parse.jstor.xml; - -import org.w3c.dom.Node; - -import edu.asu.diging.citesphere.importer.core.model.impl.ArticleMeta; -import edu.asu.diging.citesphere.importer.core.model.impl.ContainerMeta; - -public interface IBibEntryParser { - - void parseJournalMetaTag(Node node, ContainerMeta meta); - - void parseArticleMetaTag(Node node, ArticleMeta meta); - -} diff --git a/citesphere-importer/src/main/resources/app.properties b/citesphere-importer/src/main/resources/app.properties deleted file mode 100644 index d975229..0000000 --- a/citesphere-importer/src/main/resources/app.properties +++ /dev/null @@ -1,17 +0,0 @@ -app.url=${app.url} -app.name=${app.name} -app.createUserApprovalPath=login/reset/initiated -app.resetPasswordPath= -app.resetPasswordSubject=Password Reset request for $app -password.reset.expiration=60 - -mongo.database.name=${mongo.database.name} -mongo.database.host=${mongo.database.host} -mongo.database.port=${mongo.database.port} -mongo.database.user=${mongo.database.user} -mongo.database.password=${mongo.database.password} -mongo.database.authdb=${mongo.database.authdb} - -_citephere_oauth2_app_clientid=${citesphere.oauth2.app.clientId} - -giles_baseurl=${giles.baseurl} \ No newline at end of file diff --git a/citesphere-importer/src/main/resources/config.properties b/citesphere-importer/src/main/resources/config.properties index db2da3d..2388af7 100644 --- a/citesphere-importer/src/main/resources/config.properties +++ b/citesphere-importer/src/main/resources/config.properties @@ -27,5 +27,6 @@ _zotero_base_url=https://api.zotero.org/ _zotero_template_api_endpoint=/items/new?itemType= _zotero_create_item_api_endpoint=/groups/%s/items -giles_upload_endpoint=/api/v2/files/upload +_giles_baseurl=${giles.baseurl} +_giles_upload_endpoint=/api/v2/files/upload From 7c5d730f669384ba2b473983cf2ea05cdcee11b4 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Thu, 27 Feb 2025 11:45:41 -0700 Subject: [PATCH 45/49] [CITE-163] Removed unwanted changes --- .../citesphere/importer/core/service/ICitesphereConnector.java | 2 +- .../importer/core/service/impl/CitesphereConnector.java | 3 ++- citesphere-importer/src/main/resources/config.properties | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java index dc81f68..0f47939 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/ICitesphereConnector.java @@ -8,5 +8,5 @@ public interface ICitesphereConnector { JobInfo getJobInfo(String apiToken) throws CitesphereCommunicationException; String getUploadeFile(String apiToken) throws CitesphereCommunicationException; - + } \ No newline at end of file diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java index 8efd2d3..93430ac 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CitesphereConnector.java @@ -9,11 +9,13 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Arrays; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.UUID; import javax.annotation.PostConstruct; +import javax.print.attribute.standard.DateTimeAtCompleted; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; @@ -262,5 +264,4 @@ private boolean refreshToken(String errorMessage) { return false; } - } diff --git a/citesphere-importer/src/main/resources/config.properties b/citesphere-importer/src/main/resources/config.properties index 2388af7..df90288 100644 --- a/citesphere-importer/src/main/resources/config.properties +++ b/citesphere-importer/src/main/resources/config.properties @@ -9,8 +9,6 @@ _citesphere_download_path=${citesphere.download.path} _citesphere_token_endpoint=api/oauth/token?grant_type=client_credentials _citesphere_upload_path=api/v1/upload _citesphere_job_info_path=api/v1/job/info -_citesphere_get_item_path=api/v1/groups/{groupId}/items/{item} -_citesphere_upload_file_path=api/v1/groups/{groupId}/items/{item}/file _zotero_client_key=${zotero.client.key} _zotero_client_secret=${zotero.client.secret} From 4aac5461dcdb86aad7ce3a813432e80f80fbf50b Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 28 Feb 2025 14:48:04 -0700 Subject: [PATCH 46/49] [CITE-163] Resolving code factor issues --- .../kafka/impl/ReferenceImportListener.java | 1 - .../impl/CollectionImportProcessor.java | 3 - .../parse/iterators/BibFileIterator.java | 1 - .../template/impl/ExtraFieldHelper.java | 112 +++++++++--------- 4 files changed, 58 insertions(+), 59 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java index 82f7e92..6b1deb4 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java @@ -49,7 +49,6 @@ public void receiveCollectionImportMessage(String message) { msg = mapper.readValue(message, KafkaJobMessage.class); } catch (IOException e) { logger.error("Could not unmarshall message.", e); - // FIXME: handle this case return; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java index f431c3f..2d5525a 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java @@ -152,7 +152,6 @@ private void sendMessage(ItemCreationResponse message, String jobId, Status stat try { requestProducer.sendRequest(returnMessage, KafkaTopics.REFERENCES_IMPORT_DONE_TOPIC); } catch (MessageCreationException e) { - // FIXME handle this case logger.error("Exception sending message.", e); } } @@ -183,7 +182,6 @@ private JobInfo getJobInfo(KafkaJobMessage message) { try { info = connector.getJobInfo(message.getId()); } catch (CitesphereCommunicationException e) { - // FIXME this needs to be handled better logger.error("Could not get Zotero info.", e); return null; } @@ -195,7 +193,6 @@ private String downloadFile(KafkaJobMessage message) { try { file = connector.getUploadeFile(message.getId()); } catch (CitesphereCommunicationException e) { - // FIXME this needs to be handled better logger.error("Could not get Zotero info.", e); return null; } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index d8f8d77..15194cb 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -342,7 +342,6 @@ public boolean hasNext() { @Override public void close() { - // TODO Auto-generated method stub } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java index 83fa52a..89f90cd 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java @@ -357,64 +357,68 @@ public void createReferences(BibEntry article, ObjectNode root) { ArrayNode references = root.putArray("references"); for (Reference ref : article.getArticleMeta().getReferences()) { ObjectNode refNode = references.addObject(); - if (ref.getAuthorString() != null) { - refNode.put("authorString", ref.getAuthorString()); - } - if (ref.getContributors() != null) { - ArrayNode contributors = refNode.putArray("contributors"); - int idx = 0; - for (Contributor contrib : ref.getContributors()) { - ObjectNode contribNode = contributors.addObject(); - fillPerson(contrib, contribNode, idx); - idx++; - } - } - if (ref.getTitle() != null) { - refNode.put("title", ref.getTitle()); - } - if (ref.getEndPage() != null) { - refNode.put("endPage", ref.getEndPage()); - } - if (ref.getFirstPage() != null) { - refNode.put("firstPage", ref.getFirstPage()); - } - if (ref.getIdentifier() != null) { - refNode.put("identifier", ref.getIdentifier()); - } - if (ref.getIdentifierType() != null) { - refNode.put("identifierType", ref.getIdentifierType()); - } - if (ref.getReferenceString() != null) { - refNode.put("referenceString", ref.getReferenceString()); - } - if (ref.getReferenceStringRaw() != null) { - refNode.put("referenceStringRaw", ref.getReferenceStringRaw()); - } - if (ref.getSource() != null) { - refNode.put("source", ref.getSource()); - } - if (ref.getVolume() != null) { - refNode.put("volume", ref.getVolume()); - } - if (ref.getYear() != null) { - refNode.put("year", ref.getYear()); - } - if (ref.getPublicationType() != null) { - refNode.put("publicationType", ref.getPublicationType()); - } - if (ref.getCitationId()!= null) { - refNode.put("citationId", ref.getCitationId()); - } - if (ref.getReferenceId() != null) { - refNode.put("referenceId", ref.getReferenceId()); - } - if (ref.getReferenceLabel() != null) { - refNode.put("referenceLabel", ref.getReferenceLabel()); - } + createSingleReference(ref, refNode); } } } + private void createSingleReference(Reference ref, ObjectNode refNode) { + if (ref.getAuthorString() != null) { + refNode.put("authorString", ref.getAuthorString()); + } + if (ref.getContributors() != null) { + ArrayNode contributors = refNode.putArray("contributors"); + int idx = 0; + for (Contributor contrib : ref.getContributors()) { + ObjectNode contribNode = contributors.addObject(); + fillPerson(contrib, contribNode, idx); + idx++; + } + } + if (ref.getTitle() != null) { + refNode.put("title", ref.getTitle()); + } + if (ref.getEndPage() != null) { + refNode.put("endPage", ref.getEndPage()); + } + if (ref.getFirstPage() != null) { + refNode.put("firstPage", ref.getFirstPage()); + } + if (ref.getIdentifier() != null) { + refNode.put("identifier", ref.getIdentifier()); + } + if (ref.getIdentifierType() != null) { + refNode.put("identifierType", ref.getIdentifierType()); + } + if (ref.getReferenceString() != null) { + refNode.put("referenceString", ref.getReferenceString()); + } + if (ref.getReferenceStringRaw() != null) { + refNode.put("referenceStringRaw", ref.getReferenceStringRaw()); + } + if (ref.getSource() != null) { + refNode.put("source", ref.getSource()); + } + if (ref.getVolume() != null) { + refNode.put("volume", ref.getVolume()); + } + if (ref.getYear() != null) { + refNode.put("year", ref.getYear()); + } + if (ref.getPublicationType() != null) { + refNode.put("publicationType", ref.getPublicationType()); + } + if (ref.getCitationId()!= null) { + refNode.put("citationId", ref.getCitationId()); + } + if (ref.getReferenceId() != null) { + refNode.put("referenceId", ref.getReferenceId()); + } + if (ref.getReferenceLabel() != null) { + refNode.put("referenceLabel", ref.getReferenceLabel()); + } + } + public void createReferenceCount(BibEntry article, ObjectNode root) { if (article.getArticleMeta().getReferenceCount() != null) { root.put("referenceCount", article.getArticleMeta().getReferenceCount()); From 8d64005063c02238f2bf85185c31b635ed774345 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 28 Feb 2025 15:05:08 -0700 Subject: [PATCH 47/49] [CITE-163] Resolving code factor issues --- .../template/impl/ExtraFieldHelper.java | 86 +++++++------------ 1 file changed, 31 insertions(+), 55 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java index 89f90cd..118e7a4 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/zotero/template/impl/ExtraFieldHelper.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import javax.annotation.PostConstruct; @@ -357,65 +358,40 @@ public void createReferences(BibEntry article, ObjectNode root) { ArrayNode references = root.putArray("references"); for (Reference ref : article.getArticleMeta().getReferences()) { ObjectNode refNode = references.addObject(); - createSingleReference(ref, refNode); + + if (ref.getContributors() != null) { + ArrayNode contributors = refNode.putArray("contributors"); + int idx = 0; + for (Contributor contrib : ref.getContributors()) { + ObjectNode contribNode = contributors.addObject(); + fillPerson(contrib, contribNode, idx); + idx++; + } + } + + putIfNotNull(ref, Reference::getAuthorString, "authorString", refNode); + putIfNotNull(ref, Reference::getTitle, "title", refNode); + putIfNotNull(ref, Reference::getEndPage, "endPage", refNode); + putIfNotNull(ref, Reference::getFirstPage, "firstPage", refNode); + putIfNotNull(ref, Reference::getIdentifier, "identifier", refNode); + putIfNotNull(ref, Reference::getIdentifierType, "identifierType", refNode); + putIfNotNull(ref, Reference::getReferenceString, "referenceString", refNode); + putIfNotNull(ref, Reference::getReferenceStringRaw, "referenceStringRaw", refNode); + putIfNotNull(ref, Reference::getSource, "source", refNode); + putIfNotNull(ref, Reference::getVolume, "volume", refNode); + putIfNotNull(ref, Reference::getYear, "year", refNode); + putIfNotNull(ref, Reference::getPublicationType, "publicationType", refNode); + putIfNotNull(ref, Reference::getCitationId, "citationId", refNode); + putIfNotNull(ref, Reference::getReferenceId, "referenceId", refNode); + putIfNotNull(ref, Reference::getReferenceLabel, "referenceLabel", refNode); } } } - private void createSingleReference(Reference ref, ObjectNode refNode) { - if (ref.getAuthorString() != null) { - refNode.put("authorString", ref.getAuthorString()); - } - if (ref.getContributors() != null) { - ArrayNode contributors = refNode.putArray("contributors"); - int idx = 0; - for (Contributor contrib : ref.getContributors()) { - ObjectNode contribNode = contributors.addObject(); - fillPerson(contrib, contribNode, idx); - idx++; - } - } - if (ref.getTitle() != null) { - refNode.put("title", ref.getTitle()); - } - if (ref.getEndPage() != null) { - refNode.put("endPage", ref.getEndPage()); - } - if (ref.getFirstPage() != null) { - refNode.put("firstPage", ref.getFirstPage()); - } - if (ref.getIdentifier() != null) { - refNode.put("identifier", ref.getIdentifier()); - } - if (ref.getIdentifierType() != null) { - refNode.put("identifierType", ref.getIdentifierType()); - } - if (ref.getReferenceString() != null) { - refNode.put("referenceString", ref.getReferenceString()); - } - if (ref.getReferenceStringRaw() != null) { - refNode.put("referenceStringRaw", ref.getReferenceStringRaw()); - } - if (ref.getSource() != null) { - refNode.put("source", ref.getSource()); - } - if (ref.getVolume() != null) { - refNode.put("volume", ref.getVolume()); - } - if (ref.getYear() != null) { - refNode.put("year", ref.getYear()); - } - if (ref.getPublicationType() != null) { - refNode.put("publicationType", ref.getPublicationType()); - } - if (ref.getCitationId()!= null) { - refNode.put("citationId", ref.getCitationId()); - } - if (ref.getReferenceId() != null) { - refNode.put("referenceId", ref.getReferenceId()); - } - if (ref.getReferenceLabel() != null) { - refNode.put("referenceLabel", ref.getReferenceLabel()); + private void putIfNotNull(Reference ref, Function getter, String key, ObjectNode refNode) { + String value = getter.apply(ref); + if (value != null) { + refNode.put(key, value); } } From 771ba50e6c359e3a1d02e68e31a226b119d44267 Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 2 May 2025 16:33:37 -0700 Subject: [PATCH 48/49] [CITE-163] Addressed PR comments --- .../parse/iterators/BibFileIterator.java | 114 ++++++++++++------ 1 file changed, 79 insertions(+), 35 deletions(-) diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index 15194cb..d0976cd 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -1,11 +1,14 @@ package edu.asu.diging.citesphere.importer.core.service.parse.iterators; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -13,13 +16,22 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.io.FileUtils; +import org.jbibtex.BibTeXDatabase; +import org.jbibtex.BibTeXEntry; +import org.jbibtex.BibTeXParser; +import org.jbibtex.DigitStringValue; +import org.jbibtex.Key; +import org.jbibtex.ObjectResolutionException; +import org.jbibtex.ParseException; +import org.jbibtex.StringValue; +import org.jbibtex.TokenMgrException; +import org.jbibtex.Value; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.social.zotero.api.Data; import org.springframework.social.zotero.api.Item; -import edu.asu.diging.citesphere.factory.impl.ParseExtra; +import edu.asu.diging.citesphere.factory.impl.ExtraParser; import edu.asu.diging.citesphere.importer.core.model.BibEntry; import edu.asu.diging.citesphere.importer.core.model.impl.Affiliation; import edu.asu.diging.citesphere.importer.core.model.impl.ArticleId; @@ -32,7 +44,6 @@ import edu.asu.diging.citesphere.importer.core.model.impl.Publication; import edu.asu.diging.citesphere.importer.core.model.impl.Reference; import edu.asu.diging.citesphere.importer.core.service.IGilesConnector; -import edu.asu.diging.citesphere.importer.core.service.giles.impl.GilesConnector; import edu.asu.diging.citesphere.importer.core.service.impl.JobInfo; import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; import edu.asu.diging.citesphere.model.bib.IAffiliation; @@ -48,13 +59,13 @@ public class BibFileIterator implements BibEntryIterator { private final Logger logger = LoggerFactory.getLogger(getClass()); - private ParseExtra parseExtra; + private ExtraParser extraParser; private String filePath; private String groupId; private String collectionId; private JobInfo info; - private Iterator lineIterator; + private Iterator bibIterator; private Map typeMap; private IGilesConnector gilesConnector; @@ -64,17 +75,17 @@ public BibFileIterator(String filePath, JobInfo info, IGilesConnector gilesConne this.collectionId = info.getCollectionId(); this.info = info; this.gilesConnector = gilesConnector; - parseExtra = new ParseExtra(); - parseExtra.init(); + extraParser = new ExtraParser(); + extraParser.init(); init(); } private void init() { - try { - lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); - } catch (IOException e) { - logger.error("Could not create line iterator.", e); - } +// try { +// lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); +// } catch (IOException e) { +// logger.error("Could not create line iterator.", e); +// } typeMap = new HashMap(); typeMap.put("article", Publication.ARTICLE); typeMap.put("book", Publication.BOOK); @@ -85,34 +96,67 @@ private void init() { typeMap.put("research-article", Publication.ARTICLE); typeMap.put("book-review", Publication.REVIEW); typeMap.put("patent", Publication.PROCEEDINGS_PAPER); + + try (FileReader reader = new FileReader(filePath)) { + BibTeXParser parser = new BibTeXParser(); + BibTeXDatabase db = parser.parse(reader); + Collection entries = db.getEntries().values(); + bibIterator = entries.iterator(); + } catch (IOException | ObjectResolutionException | TokenMgrException | ParseException e) { + logger.error("Could not parse .bib file: " + filePath, e); + bibIterator = Collections.emptyIterator(); + } } @Override public BibEntry next() { - BibEntry entry = new Publication(); +// BibEntry entry = new Publication(); +// Map fields = new HashMap<>(); +// while (lineIterator.hasNext()) { +// String line = lineIterator.next().trim(); +// if(!line.isBlank() && line.charAt(0)=='@') { +// entry.setArticleType(typeMap.get(line.substring(1, line.indexOf('{')))); +// } else if (line.equals("}")) { +// entry.setJournalMeta(parseJournalMeta(fields)); +// entry.setArticleMeta(parseArticleMeta(fields)); +// fields.clear(); +// break; +// } else if (line.contains("=")) { +// String[] parts = line.split("=", 2); +// if (parts.length == 2) { +// String key = parts[0].trim(); +// String value = parts[1].trim(); +// if (value.endsWith(",")) { +// value = value.substring(0, value.length()-1).replaceAll("^\\{|\\}$", ""); +// } +// fields.put(key, value); +// } +// } +// } +// +// return entry; + BibTeXEntry bib = bibIterator.next(); + Publication entry = new Publication(); + + String rawType = bib.getType().getValue().toLowerCase(); + entry.setArticleType(typeMap.getOrDefault(rawType, Publication.ARTICLE)); + Map fields = new HashMap<>(); - while (lineIterator.hasNext()) { - String line = lineIterator.next().trim(); - if(!line.isBlank() && line.charAt(0)=='@') { - entry.setArticleType(typeMap.get(line.substring(1, line.indexOf('{')))); - } else if (line.equals("}")) { - entry.setJournalMeta(parseJournalMeta(fields)); - entry.setArticleMeta(parseArticleMeta(fields)); - fields.clear(); - break; - } else if (line.contains("=")) { - String[] parts = line.split("=", 2); - if (parts.length == 2) { - String key = parts[0].trim(); - String value = parts[1].trim(); - if (value.endsWith(",")) { - value = value.substring(0, value.length()-1).replaceAll("^\\{|\\}$", ""); - } - fields.put(key, value); - } + for (Map.Entry f : bib.getFields().entrySet()) { + String name = f.getKey().getValue().toLowerCase(); + Value val = f.getValue(); + String str = ""; + if (val instanceof StringValue) { + str = ((StringValue) val).getString(); + } else if (val instanceof DigitStringValue) { + str = ((DigitStringValue) val).getString(); } + fields.put(name, str); } + entry.setJournalMeta(parseJournalMeta(fields)); + entry.setArticleMeta(parseArticleMeta(fields)); + return entry; } @@ -183,8 +227,8 @@ private ArticleMeta parseArticleMeta(Map fields) { Set creators = new HashSet<>(); citation.setOtherCreators(creators); - parseExtra.parseMetaDataNote(citation, item); - parseExtra.parseExtra(data, citation); + extraParser.parseMetaDataNote(citation, item); + extraParser.parseExtra(data, citation); List collectionIds = new ArrayList<>(); if (collectionId != null && !collectionId.trim().isEmpty()) { @@ -337,7 +381,7 @@ private IGilesUpload createGilesUpload(String gilesFilePath, JobInfo info) { @Override public boolean hasNext() { - return lineIterator.hasNext(); + return bibIterator.hasNext(); } @Override From e089542ff6c34b33f489dd70607c42c71be8dc5c Mon Sep 17 00:00:00 2001 From: PradnyaC11 Date: Fri, 9 May 2025 16:24:36 -0700 Subject: [PATCH 49/49] [CITE-163] Addressed PR comments --- citesphere-importer/pom.xml | 21 +- .../kafka/impl/ReferenceImportListener.java | 6 +- .../impl/CollectionImportProcessor.java | 202 ------------------ .../core/service/impl/ImportProcessor.java | 9 +- .../parse/iterators/BibFileIterator.java | 30 --- 5 files changed, 13 insertions(+), 255 deletions(-) delete mode 100644 citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java diff --git a/citesphere-importer/pom.xml b/citesphere-importer/pom.xml index ad306b9..5ccb826 100644 --- a/citesphere-importer/pom.xml +++ b/citesphere-importer/pom.xml @@ -17,7 +17,6 @@ 2.2.6.RELEASE 0.6 1.24 - 2.4.0.RELEASE $2a$04$oQo44vqcDIFRoYKiAXoNheurzkwX9dcNmowvTX/hsWuBMwijqn44i @@ -128,12 +127,6 @@ spring-aop - - org.springframework.security.oauth - spring-security-oauth2 - ${spring-security-oauth2-version} - - org.aspectj aspectjrt @@ -261,8 +254,12 @@ 2.2.11 - - + + + org.jbibtex + jbibtex + 1.0.20 + @@ -315,12 +312,6 @@ test - - org.springframework - spring-test - test - - com.opencsv opencsv diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java index 6b1deb4..bd9881f 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/kafka/impl/ReferenceImportListener.java @@ -21,10 +21,6 @@ public class ReferenceImportListener { @Autowired @Qualifier("importProcessor") private IImportProcessor fileProcessor; - - @Autowired - @Qualifier("collectionImportProcessor") - private IImportProcessor collectionProcessor; @KafkaListener(topics = KafkaTopics.REFERENCES_IMPORT_TOPIC) public void receiveMessage(String message) { @@ -52,7 +48,7 @@ public void receiveCollectionImportMessage(String message) { return; } - collectionProcessor.process(msg); + fileProcessor.process(msg); } } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java deleted file mode 100644 index 2d5525a..0000000 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/CollectionImportProcessor.java +++ /dev/null @@ -1,202 +0,0 @@ -package edu.asu.diging.citesphere.importer.core.service.impl; - -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; - -import javax.annotation.PostConstruct; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import edu.asu.diging.citesphere.importer.core.exception.CitesphereCommunicationException; -import edu.asu.diging.citesphere.importer.core.exception.IteratorCreationException; -import edu.asu.diging.citesphere.importer.core.exception.MessageCreationException; -import edu.asu.diging.citesphere.importer.core.kafka.impl.KafkaRequestProducer; -import edu.asu.diging.citesphere.importer.core.model.BibEntry; -import edu.asu.diging.citesphere.importer.core.model.ItemType; -import edu.asu.diging.citesphere.importer.core.model.impl.Publication; -import edu.asu.diging.citesphere.importer.core.service.ICitesphereConnector; -import edu.asu.diging.citesphere.importer.core.service.IImportProcessor; -import edu.asu.diging.citesphere.importer.core.service.parse.BibEntryIterator; -import edu.asu.diging.citesphere.importer.core.service.parse.IHandlerRegistry; -import edu.asu.diging.citesphere.importer.core.zotero.IZoteroConnector; -import edu.asu.diging.citesphere.importer.core.zotero.template.IJsonGenerationService; -import edu.asu.diging.citesphere.messages.KafkaTopics; -import edu.asu.diging.citesphere.messages.model.ItemCreationResponse; -import edu.asu.diging.citesphere.messages.model.KafkaImportReturnMessage; -import edu.asu.diging.citesphere.messages.model.KafkaJobMessage; -import edu.asu.diging.citesphere.messages.model.ResponseCode; -import edu.asu.diging.citesphere.messages.model.Status; - -@Service -public class CollectionImportProcessor implements IImportProcessor { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Autowired - private ICitesphereConnector connector; - - @Autowired - private IHandlerRegistry handlerRegistry; - - @Autowired - private IZoteroConnector zoteroConnector; - - @Autowired - private IJsonGenerationService generationService; - - @Autowired - private KafkaRequestProducer requestProducer; - - private Map itemTypeMapping = new HashMap<>(); - - @PostConstruct - public void init() { - // this needs to be changed and improved, but for now it works - itemTypeMapping.put(Publication.ARTICLE, ItemType.JOURNAL_ARTICLE); - itemTypeMapping.put(Publication.BOOK, ItemType.BOOK); - itemTypeMapping.put(Publication.BOOK_CHAPTER, ItemType.BOOK_SECTION); - itemTypeMapping.put(Publication.LETTER, ItemType.LETTER); - itemTypeMapping.put(Publication.NEWS_ITEM, ItemType.NEWSPAPER_ARTICLE); - itemTypeMapping.put(Publication.PROCEEDINGS_PAPER, ItemType.CONFERENCE_PAPER); - itemTypeMapping.put(Publication.DOCUMENT, ItemType.DOCUMENT); - } - - /* - * (non-Javadoc) - * - * @see - * edu.asu.diging.citesphere.importer.core.service.impl.IImportProcessor#process - * (edu.asu.diging.citesphere.importer.core.kafka.impl.KafkaJobMessage) - */ - @Override - public void process(KafkaJobMessage message) { - JobInfo info = getJobInfo(message); - if (info == null) { - sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X10); - return; - } - - String filePath = downloadFile(message); - if (filePath == null) { - sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X20); - return; - } - - sendMessage(null, message.getId(), Status.PROCESSING, ResponseCode.P00); - BibEntryIterator bibIterator = null; - try { - bibIterator = handlerRegistry.handleFile(info, filePath); - } catch (IteratorCreationException e1) { - logger.error("Could not create iterator.", e1); - } - - if (bibIterator == null) { - sendMessage(null, message.getId(), Status.FAILED, ResponseCode.X30); - return; - } - - ObjectMapper mapper = new ObjectMapper(); - ArrayNode root = mapper.createArrayNode(); - int entryCounter = 0; - Map filesMap = new HashMap<>(); - while (bibIterator.hasNext()) { - BibEntry entry = bibIterator.next(); - if (entry.getArticleType() == null) { - // something is wrong with this entry, let's ignore it - continue; - } - ItemType type = itemTypeMapping.get(entry.getArticleType()); - JsonNode template = zoteroConnector.getTemplate(type); - ObjectNode bibNode = generationService.generateJson(template, entry); - if(entry.getArticleMeta().getFilePath() != null) { - filesMap.put(entry.getArticleMeta().getArticleTitle(), entry.getArticleMeta().getFilePath()); - } - root.add(bibNode); - entryCounter++; - - // we can submit max 50 entries to Zotoro - if (entryCounter >= 50) { - ItemCreationResponse response = submitEntries(root, info); - entryCounter = 0; - root = mapper.createArrayNode(); - } - - } - - bibIterator.close(); - - ItemCreationResponse response = null; - if (entryCounter > 0) { - response = submitEntries(root, info); - } - - response = response != null ? response : new ItemCreationResponse(); - sendMessage(response, message.getId(), Status.DONE, ResponseCode.S00); - - } - - private void sendMessage(ItemCreationResponse message, String jobId, Status status, ResponseCode code) { - KafkaImportReturnMessage returnMessage = new KafkaImportReturnMessage(message, jobId); - returnMessage.setStatus(status); - returnMessage.setCode(code); - try { - requestProducer.sendRequest(returnMessage, KafkaTopics.REFERENCES_IMPORT_DONE_TOPIC); - } catch (MessageCreationException e) { - logger.error("Exception sending message.", e); - } - } - - private ItemCreationResponse submitEntries(ArrayNode entries, JobInfo info) { - ObjectMapper mapper = new ObjectMapper(); - try { - String msg = mapper.writeValueAsString(entries); - logger.info("Submitting " + msg); - ItemCreationResponse response = zoteroConnector.addEntries(info, entries); - if (response != null) { - logger.info(response.getSuccessful() + ""); - logger.error(response.getFailed() + ""); - } else { - logger.error("Item creation failed."); - } - return response; - } catch (URISyntaxException e) { - logger.error("Could not store new entry.", e); - } catch (JsonProcessingException e) { - logger.error("Could not write JSON."); - } - return null; - } - - private JobInfo getJobInfo(KafkaJobMessage message) { - JobInfo info = null; - try { - info = connector.getJobInfo(message.getId()); - } catch (CitesphereCommunicationException e) { - logger.error("Could not get Zotero info.", e); - return null; - } - return info; - } - - private String downloadFile(KafkaJobMessage message) { - String file = null; - try { - file = connector.getUploadeFile(message.getId()); - } catch (CitesphereCommunicationException e) { - logger.error("Could not get Zotero info.", e); - return null; - } - return file; - } - -} diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java index b4c5a9b..966d116 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/impl/ImportProcessor.java @@ -120,6 +120,8 @@ public void process(KafkaJobMessage message) { ObjectMapper mapper = new ObjectMapper(); ArrayNode root = mapper.createArrayNode(); int entryCounter = 0; + Map filesMap = new HashMap<>(); + ItemCreationResponse response = null; while (bibIterator.hasNext()) { BibEntry entry = bibIterator.next(); if (entry.getArticleType() == null) { @@ -129,13 +131,15 @@ public void process(KafkaJobMessage message) { ItemType type = itemTypeMapping.get(entry.getArticleType()); JsonNode template = zoteroConnector.getTemplate(type); ObjectNode bibNode = generationService.generateJson(template, entry); - + if(entry.getArticleMeta().getFilePath() != null) { + filesMap.put(entry.getArticleMeta().getArticleTitle(), entry.getArticleMeta().getFilePath()); + } root.add(bibNode); entryCounter++; // we can submit max 50 entries to Zotoro if (entryCounter >= 50) { - submitEntries(root, info); + response = submitEntries(root, info); entryCounter = 0; root = mapper.createArrayNode(); } @@ -144,7 +148,6 @@ public void process(KafkaJobMessage message) { bibIterator.close(); - ItemCreationResponse response = null; if (entryCounter > 0) { response = submitEntries(root, info); } diff --git a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java index d0976cd..e7ebff8 100644 --- a/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java +++ b/citesphere-importer/src/main/java/edu/asu/diging/citesphere/importer/core/service/parse/iterators/BibFileIterator.java @@ -81,11 +81,6 @@ public BibFileIterator(String filePath, JobInfo info, IGilesConnector gilesConne } private void init() { -// try { -// lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8"); -// } catch (IOException e) { -// logger.error("Could not create line iterator.", e); -// } typeMap = new HashMap(); typeMap.put("article", Publication.ARTICLE); typeMap.put("book", Publication.BOOK); @@ -110,31 +105,6 @@ private void init() { @Override public BibEntry next() { -// BibEntry entry = new Publication(); -// Map fields = new HashMap<>(); -// while (lineIterator.hasNext()) { -// String line = lineIterator.next().trim(); -// if(!line.isBlank() && line.charAt(0)=='@') { -// entry.setArticleType(typeMap.get(line.substring(1, line.indexOf('{')))); -// } else if (line.equals("}")) { -// entry.setJournalMeta(parseJournalMeta(fields)); -// entry.setArticleMeta(parseArticleMeta(fields)); -// fields.clear(); -// break; -// } else if (line.contains("=")) { -// String[] parts = line.split("=", 2); -// if (parts.length == 2) { -// String key = parts[0].trim(); -// String value = parts[1].trim(); -// if (value.endsWith(",")) { -// value = value.substring(0, value.length()-1).replaceAll("^\\{|\\}$", ""); -// } -// fields.put(key, value); -// } -// } -// } -// -// return entry; BibTeXEntry bib = bibIterator.next(); Publication entry = new Publication();