Skip to content

Commit

Permalink
Fix #4756: Add support for logging an invested engagement event (#4757)
Browse files Browse the repository at this point in the history
## Explanation
Fixes #4756

This PR introduces a new event for tracking individual play sessions
where a learner has reached a level of 'invested' engagement, where
'invested' here is considered to be strong engagement with a likelihood
of continuing at least with that play session.

This metric is planned to be used as one of the team's conversion
metrics to better help track the user marketing pipeline by helping to
determine how we can better reach learners who are more likely to reach
this level of engagement with lessons (and, thus, hopefully learn what
they need to).

Note that the event is based on a single play session, not a profile or
even a single exploration (so if a user pauses and resumes an
exploration, the count for engagement resets **from that point**).
Engagement means completing _and_ moving past at minimum 3 cards (which
may just be simple 'Continue' button interactions).

For simplicity, this PR keeps the new event name the same between the
Kenya & non-Kenya styles of naming events.

I've verified that the event is logging as expected using Firebase's
DebugView:


![image](https://user-images.githubusercontent.com/12983742/203501899-dbccd386-4e04-4966-82bc-a4646b3b742c.png)

## Essential Checklist
- [x] The PR title and explanation each start with "Fix #bugnum: " (If
this PR fixes part of an issue, prefix the title with "Fix part of
#bugnum: ...".)
- [x] Any changes to
[scripts/assets](https://github.com/oppia/oppia-android/tree/develop/scripts/assets)
files have their rationale included in the PR explanation.
- [x] The PR follows the [style
guide](https://github.com/oppia/oppia-android/wiki/Coding-style-guide).
- [x] The PR does not contain any unnecessary code changes from Android
Studio
([reference](https://github.com/oppia/oppia-android/wiki/Guidance-on-submitting-a-PR#undo-unnecessary-changes)).
- [x] The PR is made from a branch that's **not** called "develop" and
is up-to-date with "develop".
- [x] The PR is **assigned** to the appropriate reviewers
([reference](https://github.com/oppia/oppia-android/wiki/Guidance-on-submitting-a-PR#clarification-regarding-assignees-and-reviewers-section)).

## For UI-specific PRs only
This PR is introducing a log and has no impact on the UI or UX of the
app.
  • Loading branch information
BenHenning committed Nov 23, 2022
1 parent 0818246 commit eb96635
Show file tree
Hide file tree
Showing 12 changed files with 393 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1046,6 +1046,9 @@ class ExplorationProgressController @Inject constructor(
private var helpIndex = HelpIndex.getDefaultInstance()
private var availableCardCount: Int = -1

private var hasReachedInvestedEngagement = false
private var completedStateCount = 0

/**
* The [LearnerAnalyticsLogger.ExplorationAnalyticsLogger] to be used for logging
* exploration-specific events.
Expand Down Expand Up @@ -1087,6 +1090,13 @@ class ExplorationProgressController @Inject constructor(

// Force the card count to update.
availableCardCount = explorationProgress.stateDeck.getViewedStateCount()

if (!hasReachedInvestedEngagement &&
completedStateCount >= MINIMUM_COMPLETED_STATE_COUNT_FOR_INVESTED_ENGAGEMENT
) {
it.logInvestedEngagement()
hasReachedInvestedEngagement = true
}
}
}

Expand All @@ -1106,6 +1116,7 @@ class ExplorationProgressController @Inject constructor(
fun endState() {
stateAnalyticsLogger?.logEndCard()
explorationAnalyticsLogger.endCard()
completedStateCount++
}

/** Checks and logs for hint-based changes based on the provided [HelpIndex]. */
Expand Down Expand Up @@ -1279,6 +1290,8 @@ class ExplorationProgressController @Inject constructor(
}

private companion object {
private const val MINIMUM_COMPLETED_STATE_COUNT_FOR_INVESTED_ENGAGEMENT = 3

/**
* Returns a collectable [Flow] that notifies [collector] for this [StateFlow]s initial state,
* and every change after.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,15 @@ class LearnerAnalyticsLogger @Inject constructor(
logStateEvent(contentId, ::createPlayVoiceOverContext, EventBuilder::setPlayVoiceOverContext)
}

/**
* Logs that the learner has demonstrated an invested engagement in the lesson (that is, they've
* played far enough in the lesson to indicate that they're not just quickly browsing & then
* leaving).
*/
fun logInvestedEngagement() {
logStateEvent(EventBuilder::setReachInvestedEngagement)
}

private fun logStateEvent(setter: EventBuilder.(ExplorationContext) -> EventBuilder) =
logStateEvent(Unit, { _, context -> context }, setter)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import org.oppia.android.app.model.EphemeralState
import org.oppia.android.app.model.EphemeralState.StateTypeCase.COMPLETED_STATE
import org.oppia.android.app.model.EphemeralState.StateTypeCase.PENDING_STATE
import org.oppia.android.app.model.EphemeralState.StateTypeCase.TERMINAL_STATE
import org.oppia.android.app.model.EventLog.Context.ActivityContextCase.REACH_INVESTED_ENGAGEMENT
import org.oppia.android.app.model.Exploration
import org.oppia.android.app.model.ExplorationCheckpoint
import org.oppia.android.app.model.Fraction
Expand Down Expand Up @@ -1972,6 +1973,226 @@ class ExplorationProgressControllerTest {
}
}

@Test
fun testPlayNewExp_firstCard_notFinished_doesNotLogReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()

startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()

val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(hasEngagementEvent).isFalse()
}

@Test
fun testPlayNewExp_finishFirstCard_moveToSecond_doesNotLogReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()

startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()

val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(hasEngagementEvent).isFalse()
}

@Test
fun testPlayNewExp_finishThreeCards_doNotProceed_doesNotLogReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()

startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()
submitPrototypeState3Answer()

val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(hasEngagementEvent).isFalse()
}

@Test
fun testPlayNewExp_finishThreeCards_moveToFour_logsReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()

startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()
playThroughPrototypeState3AndMoveToNextState()

val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
assertThat(hasEngagementEvent).isTrue()
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
hasStateNameThat().isEqualTo("ItemSelectionMinOne")
}
}

@Test
fun testPlayNewExp_finishFourCards_moveToFive_logsReachInvestedEngagementEventOnlyOnce() {
logIntoAnalyticsReadyAdminProfile()

startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()
playThroughPrototypeState3AndMoveToNextState()
playThroughPrototypeState4AndMoveToNextState()

// The engagement event should only be logged once during a play session, even if the user
// continues past that point.
val engagementEventCount = fakeAnalyticsEventLogger.countEvents {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(engagementEventCount).isEqualTo(1)
}

@Test
fun testPlayNewExp_firstTwo_startOver_playFirst_doesNotLogReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()

// Restart the exploration.
restartExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()

// No engagement event should be logged, even though 3 total states were completed from the
// first and second sessions (cumulatively).
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(hasEngagementEvent).isFalse()
}

@Test
fun testPlayNewExp_firstTwo_startOver_playThreeAndMove_logsReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()

// Restart the exploration.
restartExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()
playThroughPrototypeState3AndMoveToNextState()

// An engagement event should be logged since the new session uniquely finished 3 states.
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
assertThat(hasEngagementEvent).isTrue()
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
hasStateNameThat().isEqualTo("ItemSelectionMinOne")
}
}

@Test
fun testResumeExp_stateOneTwoDone_finishThreeAndMoveForward_noLogReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()

// End, then resume the exploration and complete the third state.
endExploration()
val checkPoint = retrieveExplorationCheckpoint(TEST_EXPLORATION_ID_2)
resumeExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2, checkPoint)
playThroughPrototypeState3AndMoveToNextState()

// Despite the first three states now being completed, this isn't an engagement event since the
// user hasn't finished three states within *one* session.
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(hasEngagementEvent).isFalse()
}

@Test
fun testResumeExp_stateOneTwoDone_finishThreeMoreAndMove_logsReachInvestedEngagementEvent() {
logIntoAnalyticsReadyAdminProfile()
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()

// End, then resume the exploration and complete the third state.
endExploration()
val checkPoint = retrieveExplorationCheckpoint(TEST_EXPLORATION_ID_2)
resumeExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2, checkPoint)
playThroughPrototypeState3AndMoveToNextState()
playThroughPrototypeState4AndMoveToNextState()
playThroughPrototypeState5AndMoveToNextState()

// An engagement event should be logged now since the user completed 3 new states in the current
// session.
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
assertThat(hasEngagementEvent).isTrue()
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
hasStateNameThat().isEqualTo("NumberInput")
}
}

@Test
fun testResumeExp_finishThree_thenAnotherThreeAfterResume_logsInvestedEngagementEventTwice() {
logIntoAnalyticsReadyAdminProfile()
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
waitForGetCurrentStateSuccessfulLoad()
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()
playThroughPrototypeState3AndMoveToNextState()

// End, then resume the exploration and complete the third state.
endExploration()
val checkPoint = retrieveExplorationCheckpoint(TEST_EXPLORATION_ID_2)
resumeExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2, checkPoint)
playThroughPrototypeState4AndMoveToNextState()
playThroughPrototypeState5AndMoveToNextState()
playThroughPrototypeState6AndMoveToNextState()

// Playing enough states for the engagement event before and after resuming should result in it
// being logged twice (once for each session).
val engagementEventCount = fakeAnalyticsEventLogger.countEvents {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(engagementEventCount).isEqualTo(2)
}

@Test
fun testPlayNewExp_getToEngagementEvent_playOtherExpAndDoSame_logsEngagementEventAgain() {
logIntoAnalyticsReadyAdminProfile()

// Play through the full prototype exploration twice.
playThroughPrototypeExplorationInNewSession()
playThroughPrototypeExplorationInNewSession()

// Playing through two complete exploration sessions should result in the engagement event being
// logged twice (once for each session).
val engagementEventCount = fakeAnalyticsEventLogger.countEvents {
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
}
assertThat(engagementEventCount).isEqualTo(2)
}

@Test
fun testSubmitAnswer_correctAnswer_logsEndCardAndSubmitAnswerEvents() {
logIntoAnalyticsReadyAdminProfile()
Expand Down Expand Up @@ -2325,6 +2546,12 @@ class ExplorationProgressControllerTest {
return waitForGetCurrentStateSuccessfulLoad()
}

private fun playThroughPrototypeExplorationInNewSession() {
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
playThroughPrototypeExploration()
endExploration()
}

private fun playThroughPrototypeExploration(): EphemeralState {
playThroughPrototypeState1AndMoveToNextState()
playThroughPrototypeState2AndMoveToNextState()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1400,6 +1400,30 @@ class LearnerAnalyticsLoggerTest {
assertThat(log.type).isEqualTo(Log.ERROR)
}

@Test
fun testStateAnalyticsLogger_logReachInvestedEngagement_logsStateEventWithStateName() {
val exploration5 = loadExploration(TEST_EXPLORATION_ID_5)
val expLogger = learnerAnalyticsLogger.beginExploration(exploration5)
val stateLogger = expLogger.startCard(exploration5.getStateByName(TEST_EXP_5_STATE_THREE_NAME))

stateLogger.logInvestedEngagement()

val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
assertThat(eventLog).isEssentialPriority()
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
hasTopicIdThat().isEqualTo(TEST_TOPIC_ID)
hasStoryIdThat().isEqualTo(TEST_STORY_ID)
hasExplorationIdThat().isEqualTo(TEST_EXPLORATION_ID_5)
hasSessionIdThat().isEqualTo(DEFAULT_INITIAL_SESSION_ID)
hasVersionThat().isEqualTo(5)
hasStateNameThat().isEqualTo(TEST_EXP_5_STATE_THREE_NAME)
hasLearnerDetailsThat {
hasLearnerIdThat().isEqualTo(TEST_LEARNER_ID)
hasInstallationIdThat().isEqualTo(TEST_INSTALL_ID)
}
}
}

private fun loadExploration(expId: String): Exploration {
return monitorFactory.waitForNextSuccessfulResult(
explorationDataController.getExplorationById(profileId, expId)
Expand Down
4 changes: 4 additions & 0 deletions model/src/main/proto/oppia_logger.proto
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ message EventLog {
// value here has no importance and is always 'true'.
bool open_profile_chooser = 32;

// The event being logged indicates that the user has reached an invested level of learning
// engagement in a lesson.
ExplorationContext reach_invested_engagement = 34;

// Indicates that something went wrong when trying to log a learner analytics even for the
// device corresponding to the specified device ID.
string install_id_for_failed_analytics_log = 33;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ class FakeAnalyticsEventLogger @Inject constructor() : AnalyticsEventLogger {
/** Clears all the events that are currently logged. */
fun clearAllEvents() = eventList.clear()

/** Checks if a certain event has been logged or not. */
fun hasEventLogged(eventLog: EventLog): Boolean = eventList.contains(eventLog)
/** Returns whether a certain event has been logged or not, based on the provided [predicate]. */
fun hasEventLogged(predicate: (EventLog) -> Boolean): Boolean = eventList.find(predicate) != null

/** Returns the number of logged events that match the provided [predicate]. */
fun countEvents(predicate: (EventLog) -> Boolean): Int = eventList.count(predicate)

/** Returns true if there are no events logged. */
fun noEventsPresent(): Boolean = eventList.isEmpty()
Expand Down
Loading

0 comments on commit eb96635

Please sign in to comment.