Skip to content

Commit

Permalink
MDC log dl reqUrl etc part1 IQSS#4821 IQSS#5384
Browse files Browse the repository at this point in the history 
This is incomplete as most downloads don't log to guestbook until the end via the  DownloadInstanceWriter and it is unclear how to get this info that late in the pipe.
  • Loading branch information
matthew-a-dunlap committed Jan 7, 2019
1 parent 0bfd936 commit bd7ff84
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 8 deletions.
2 changes: 1 addition & 1 deletion doc/sphinx-guides/source/installation/config.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1509,7 +1509,7 @@ Sets how long a cached metrics result is used before re-running the query for a
:MDCLogPath
+++++++++++

Sets the path where the raw Make Data Count logs are stored before being processed.
Sets the path where the raw Make Data Count logs are stored before being processed. If not set, no logs will be created for Make Data Count.

``curl -X PUT -d '/Users/admin/glassfish4/glassfish/domains/domain1/logs' http://localhost:8080/api/admin/settings/:MDCLogPath``

Expand Down
6 changes: 6 additions & 0 deletions mdc-logs/raw-mdc-2019-01-07.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#Fields: event_time client_ip session_cookie_id user_cookie_id user_id request_url identifier filename size user-agent title publisher publisher_id authors publication_date version other_id target_url publication_year
2019-01-07T15:14:51-0500 0:0:0:0:0:0:0:1 9f4209d3c177d3cb77f4d06cf3ba - :guest http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV doi:10.5072/FK2/XTT5BV - - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Dataset One - 1 Smith, Robert| Kew, Susie 2019-01-07T18:20:54Z 1 - http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV 2019
2019-01-07T15:15:15-0500 0:0:0:0:0:0:0:1 9f4209d3c177d3cb77f4d06cf3ba - :guest http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV doi:10.5072/FK2/XTT5BV - - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Dataset One - 1 Smith, Robert| Kew, Susie 2019-01-07T18:20:54Z 1 - http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV 2019
2019-01-07T15:16:04-0500 0:0:0:0:0:0:0:1 9f4209d3c177d3cb77f4d06cf3ba - :guest http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV doi:10.5072/FK2/XTT5BV - - Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Dataset One - 1 Smith, Robert| Kew, Susie 2019-01-07T18:20:54Z 1 - http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV 2019
2019-01-07T15:16:14-0500 0:0:0:0:0:0:0:1 9f4209d3c177d3cb77f4d06cf3ba - :guest http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV doi:10.5072/FK2/XTT5BV 168298bae7c-2c5bbc1a9c8c 1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Dataset One - 1 Smith, Robert| Kew, Susie 2019-01-07T18:20:54Z 1 - http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV 2019
2019-01-07T15:16:19-0500 0:0:0:0:0:0:0:1 9f4209d3c177d3cb77f4d06cf3ba - :guest http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV doi:10.5072/FK2/XTT5BV 168298bb8ce-337d8df49763 4026 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Dataset One - 1 Smith, Robert| Kew, Susie 2019-01-07T18:20:54Z 1 - http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV 2019
12 changes: 11 additions & 1 deletion src/main/java/edu/harvard/iq/dataverse/api/Access.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import javax.faces.context.FacesContext;
import javax.json.JsonArrayBuilder;
import javax.persistence.TypedQuery;
import javax.servlet.http.HttpServletRequest;

import javax.ws.rs.GET;
import javax.ws.rs.Path;
Expand All @@ -100,6 +101,7 @@


import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.DELETE;
import javax.ws.rs.ForbiddenException;
Expand Down Expand Up @@ -562,7 +564,15 @@ public void write(OutputStream os) throws IOException,
if (gbrecs != true && file.isReleased()){
GuestbookResponse gbr = guestbookResponseService.initAPIGuestbookResponse(file.getOwner(), file, session, apiTokenUser);
guestbookResponseService.save(gbr);
MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, file);
MakeDataCountEntry entry = new MakeDataCountEntry(null, dvRequestService, file);

entry.setRequestUrl("/" + uriInfo.getPath());//uriInfo.getRequestUri().toString()); //close but has http://localhost:8080
entry.setTargetUrl("/" + uriInfo.getPath());
if(null != headers.getRequestHeader("user-agent")) {
entry.setUserAgent(headers.getRequestHeader("user-agent").get(0));
}
//entry.setSessionCookieId(); //MAD: Should we be getting this? How? Maybe normal cookie instead?

mdcLogService.logEntry(entry);
}

Expand Down
12 changes: 9 additions & 3 deletions src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookResponseCommand;
import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry;
import edu.harvard.iq.dataverse.makedatacount.MakeDataCountUtil;
import java.io.File;
import java.io.FileInputStream;
import java.net.URI;
Expand Down Expand Up @@ -261,7 +260,8 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
logger.fine("writing guestbook response, for an S3 download redirect.");
Command<?> cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner());
di.getCommand().submit(cmd);
MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), di.getDataverseRequestService(), di.getGbr().getDataFile());
MakeDataCountEntry entry = new MakeDataCountEntry(null, di.getDataverseRequestService(), di.getGbr().getDataFile());
//MAD; //MAD: SET FC RELATED STUFF MANUALLY
mdcLogService.logEntry(entry);
} catch (CommandException e) {
}
Expand Down Expand Up @@ -352,7 +352,13 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
logger.fine("writing guestbook response.");
Command<?> cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner());
di.getCommand().submit(cmd);
MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), di.getDataverseRequestService(), di.getGbr().getDataFile());
MakeDataCountEntry entry = new MakeDataCountEntry(null, di.getDataverseRequestService(), di.getGbr().getDataFile());
//MAD: mabe di.dataverseRequestService / downloadInfo. or daReq
//Nah none of those seem to work
// entry.setRequestUrl(uriInfo.getRequestUri().toString());
// entry.setTargetUrl(uriInfo.getRequestUri().toString());
// entry.setUserAgent(headers.getRequestHeader("user-agent").get(0));
//entry.setSessionCookieId(); //MAD: Should we be getting this? How? Maybe normal cookie instead?
mdcLogService.logEntry(entry);
} catch (CommandException e) {}
} else {
Expand Down
11 changes: 8 additions & 3 deletions src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountLoggingServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,14 @@ public class MakeDataCountLoggingServiceBean {
//MAD: Also the file name?
public void logEntry(MakeDataCountEntry entry) {
if(systemConfig.getMDCLogPath() != null) {
LoggingUtil.saveLogFile(entry.toString(), systemConfig.getMDCLogPath(), "raw-mdc-"+new SimpleDateFormat("yyyy-MM-dd").format(new Timestamp(new Date().getTime()))+".log", LOG_HEADER);
LoggingUtil.saveLogFile(entry.toString(), systemConfig.getMDCLogPath(), getLogFileName() , LOG_HEADER);
}
}

public String getLogFileName() {
return "counter_"+new SimpleDateFormat("yyyy-MM-dd").format(new Timestamp(new Date().getTime()))+".log";
}

public static class MakeDataCountEntry {

private String eventTime;
Expand Down Expand Up @@ -72,8 +76,8 @@ public MakeDataCountEntry() {
public MakeDataCountEntry(FacesContext fc, DataverseRequestServiceBean dvRequestService, DatasetVersion publishedVersion) {
if(fc != null) {
HttpServletRequest req = (HttpServletRequest)fc.getExternalContext().getRequest();
setRequestUrl(String.valueOf(req.getRequestURL().append("?").append(req.getQueryString())));
setTargetUrl(String.valueOf(req.getRequestURL().append("?").append(req.getQueryString())));
setRequestUrl(req.getRequestURI() + "?" + req.getQueryString());
setTargetUrl(req.getRequestURI() + "?" + req.getQueryString());
setUserAgent(req.getHeader("user-agent"));
HttpSession session = (HttpSession) fc.getExternalContext().getSession(false);
setSessionCookieId(session.getId());
Expand Down Expand Up @@ -118,6 +122,7 @@ public MakeDataCountEntry(FacesContext fc, DataverseRequestServiceBean dvRequest

//Exception thrown if no published metadata exists for DataFile
//This is passed a DataFile to log the file downloaded
//MAD: Maybe remove the fc as its unused in all calls
public MakeDataCountEntry(FacesContext fc, DataverseRequestServiceBean dvRequestService, DataFile df) throws UnsupportedOperationException {
//Passing null to the base constructor creates an entry without most of the data
// which we then prune out before logging.
Expand Down

0 comments on commit bd7ff84

Please sign in to comment.