Skip to content

Commit

Permalink
Refactor user stats (#79)
Browse files Browse the repository at this point in the history
  • Loading branch information
@artandor
artandor authored Sep 18, 2022
1 parent c48112b commit c395b95
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 88 deletions.
5 changes: 5 additions & 0 deletions .env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ APP_SECRET=0ac8957b606826b223bfa28224b415ff
DATABASE_URL=postgresql://symfony:!ChangeMe!@localhost:5432/multistream?serverVersion=13&charset=utf8
###< doctrine/doctrine-bundle ###

###< elasticsearch ###
OPENSEARCH_ENDPOINT=http://elasticsearch:9200
OPENSEARCH_AUTH=
###> elasticsearch ###

###> oauth ###
OAUTH_TWITCH_CLIENT_ID=4aq04w81v8wrvwuuzqtgoulkh3kqfe
OAUTH_TWITCH_CLIENT_SECRET=
Expand Down
8 changes: 8 additions & 0 deletions config/packages/framework.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ framework:
php_errors:
log: true

http_client:
scoped_clients:
elasticsearch.client:
base_uri: '%env(OPENSEARCH_ENDPOINT)%'

# HTTP Basic authentication
auth_basic: '%env(OPENSEARCH_AUTH)%'

when@test:
framework:
test: true
Expand Down
28 changes: 22 additions & 6 deletions src/Command/GetUsersStatsCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,21 @@
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;

#[AsCommand(
name: 'app:get-users-stats',
description: 'Get users stats for all platforms and log them for logstash to collect',
)]
class GetUsersStatsCommand extends Command
{
public function __construct(private UserRepository $userRepository,
public function __construct(private UserRepository $userRepository,
private EntityManagerInterface $em,
private LoggerInterface $logstashLogger,
private LoggerInterface $logger,
) {
private LoggerInterface $logger,
private HttpClientInterface $elasticsearchClient
)
{
parent::__construct();
}

Expand Down Expand Up @@ -55,9 +58,22 @@ protected function execute(InputInterface $input, OutputInterface $output): int
'followerCount' => $followerCount,
];
$resultData['total'] = $resultData['total'] + $followerCount;
$resultData['datetime'] = (new \DateTime())->format('Y-m-d H:m:s');
}

try {
$response = $this->elasticsearchClient->request(
'POST',
'/logstash-user_stats/_doc',
[
'json' => $resultData
],
);
} catch (TransportExceptionInterface $e) {
$this->logger->log(LogLevel::CRITICAL, 'Error while posting data to elastic search : ' . $e->getMessage());
}
$this->logstashLogger->log(LogLevel::INFO, json_encode($resultData));
$io->info('Follower Count for User '.$user->getEmail().' : '.$resultData['total']);

$io->info('Follower Count for User ' . $user->getEmail() . ' : ' . $resultData['total']);
}

return Command::SUCCESS;
Expand Down
115 changes: 33 additions & 82 deletions src/Controller/StatisticsController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

class StatisticsController extends AbstractController
{
public function __construct(private ChartBuilderInterface $chartBuilder, private HttpClientInterface $client)
public function __construct(private ChartBuilderInterface $chartBuilder, private HttpClientInterface $elasticsearchClient)
{
}

Expand Down Expand Up @@ -86,37 +86,37 @@ private function transformFollowerStats(array $elasticStats): array

$isLatest = $latestData === $bucket;

if (isset($bucket['youtube']['hits']['hits'][0]['fields'])) {
$stats['youtube'][] = $bucket['youtube']['hits']['hits'][0]['fields']['data.Youtube.followerCount'][0];
if (isset($bucket['youtube']['value'])) {
$stats['youtube'][] = $bucket['youtube']['value'];
if ($isLatest) {
$stats['total_followers'] += $bucket['youtube']['hits']['hits'][0]['fields']['data.Youtube.followerCount'][0];
$stats['total_followers'] += $bucket['youtube']['value'];
}
} else {
$stats['youtube'][] = null;
}

if (isset($bucket['trovo']['hits']['hits'][0]['fields'])) {
$stats['trovo'][] = $bucket['trovo']['hits']['hits'][0]['fields']['data.Trovo.followerCount'][0];
if (isset($bucket['trovo']['value'])) {
$stats['trovo'][] = $bucket['trovo']['value'];
if ($isLatest) {
$stats['total_followers'] += $bucket['trovo']['hits']['hits'][0]['fields']['data.Trovo.followerCount'][0];
$stats['total_followers'] += $bucket['trovo']['value'];
}
} else {
$stats['trovo'][] = null;
}

if (isset($bucket['twitch']['hits']['hits'][0]['fields'])) {
$stats['twitch'][] = $bucket['twitch']['hits']['hits'][0]['fields']['data.Twitch.followerCount'][0];
if (isset($bucket['twitch']['value'])) {
$stats['twitch'][] = $bucket['twitch']['value'];
if ($isLatest) {
$stats['total_followers'] += $bucket['twitch']['hits']['hits'][0]['fields']['data.Twitch.followerCount'][0];
$stats['total_followers'] += $bucket['twitch']['value'];
}
} else {
$stats['twitch'][] = null;
}

if (isset($bucket['brime']['hits']['hits'][0]['fields'])) {
$stats['brime'][] = $bucket['brime']['hits']['hits'][0]['fields']['data.Brime.followerCount'][0];
if (isset($bucket['brime']['value'])) {
$stats['brime'][] = $bucket['brime']['value'];
if ($isLatest) {
$stats['total_followers'] += $bucket['brime']['hits']['hits'][0]['fields']['data.Brime.followerCount'][0];
$stats['total_followers'] += $bucket['brime']['value'];
}
} else {
$stats['brime'][] = null;
Expand All @@ -128,7 +128,7 @@ private function transformFollowerStats(array $elasticStats): array

private function getStatsFromElastic(): array
{
$response = $this->client->request('GET', 'http://elasticsearch:9200/*user_stats/_search', [
$response = $this->elasticsearchClient->request('GET', '/*user_stats/_search', [
'headers' => [
'Content-Type' => 'application/json',
'Accept' => 'application/json',
Expand All @@ -140,86 +140,39 @@ private function getStatsFromElastic(): array
"date_histogram": {
"field": "datetime",
"fixed_interval": "12h",
"time_zone": "Europe/Paris"
"time_zone": "Europe/Paris",
"min_doc_count": 1
},
"aggs": {
"brime": {
"top_hits": {
"fields": [
{
"field": "data.Brime.followerCount"
}
],
"_source": false,
"size": 1,
"sort": [
{
"datetime": {
"order": "desc"
}
}
]
"max": {
"field": "Brime.followerCount"
}
},
"trovo": {
"top_hits": {
"fields": [
{
"field": "data.Trovo.followerCount"
}
],
"_source": false,
"size": 1,
"sort": [
{
"datetime": {
"order": "desc"
}
}
]
"max": {
"field": "Trovo.followerCount"
}
},
"twitch": {
"top_hits": {
"fields": [
{
"field": "data.Twitch.followerCount"
}
],
"_source": false,
"size": 1,
"sort": [
{
"datetime": {
"order": "desc"
}
}
]
"max": {
"field": "Twitch.followerCount"
}
},
"youtube": {
"top_hits": {
"fields": [
{
"field": "data.Youtube.followerCount"
}
],
"_source": false,
"size": 1,
"sort": [
{
"datetime": {
"order": "desc"
}
}
]
"max": {
"field": "Youtube.followerCount"
}
}
}
}
},
"size": 0,
"fields": [
"stored_fields": [
"*"
],
"script_fields": {},
"docvalue_fields": [
{
"field": "@timestamp",
"format": "date_time"
Expand All @@ -229,11 +182,6 @@ private function getStatsFromElastic(): array
"format": "date_time"
}
],
"script_fields": {},
"stored_fields": [
"*"
],
"runtime_mappings": {},
"_source": {
"excludes": []
},
Expand All @@ -246,13 +194,16 @@ private function getStatsFromElastic(): array
"should": [
{
"match": {
"data.userId": "'.$this->getUser()->getId().'"
"userId": ' . $this->getUser()->getId() . '
}
}
],
"minimum_should_match": 1
}
},
{
"match_all": {}
},
{
"range": {
"datetime": {
Expand Down

0 comments on commit c395b95

Please sign in to comment.