Feedback

.gitignore

@@ -0,0 +1,43 @@
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+!**/src/main/**/build/
+!**/src/test/**/build/
+.kotlin
+
+### IntelliJ IDEA ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+out/
+!**/src/main/**/out/
+!**/src/test/**/out/
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store

README.md

@@ -1,3 +1,4 @@
+[![Review Assignment Due Date](https://classroom.github.com/assets/deadline-readme-button-22041afd0340ce965d47ae6ef1cefeee28c7c493a6346c4f15d667ab976d596c.svg)](https://classroom.github.com/a/uyodabcP)
 ## Лабораторная работа: Реализация MapReduce для анализа данных о продажах с ипользованием HADOOP!!!
 # Цель работы
 
@@ -45,3 +46,11 @@
 
 # Результаты
 Результатом работы является сам код, файл с результатами и экспериментальные данные по быстродействию работы написанного кода при изменении числа worker-ов / частей, на которые разбивается файл
+
+# Полезные Web UI (на хосте)
+
+- **HDFS NameNode**: `http://localhost:9870`
+- **HDFS DataNode**: `http://localhost:9864`
+- **YARN ResourceManager**: `http://localhost:8088`
+- **YARN NodeManager**: `http://localhost:8042`
+- **MapReduce JobHistory**: `http://localhost:8188`

Taskfile.yml

@@ -0,0 +1,29 @@
+version: 3
+
+tasks:
+  up:
+    desc: "Start docker compose"
+    cmds:
+      - docker compose up -d
+
+  down:
+    desc: "Stop docker compose"
+    cmds:
+      - docker compose down
+
+  csv:
+    desc: "Upload into HDFS"
+    cmds:
+      - bash scripts/hdfs_put_csv.sh ./csv /input/sales
+
+  run:
+    desc: "Run MapReduce job"
+    cmds:
+      - bash scripts/run_job.sh --input /input/sales --output /output/sales_result --reducers 2 --map-threads 4
+      - bash scripts/fetch_result.sh --output /output/sales_result --local results/result.tsv
+
+  default:
+    desc: "csv + run"
+    cmds:
+      - task csv
+      - task run

build.gradle

@@ -0,0 +1,50 @@
+plugins {
+    id 'java'
+    id 'com.github.johnrengelman.shadow' version '8.1.1'
+}
+
+group = 'ru.taymir.telecom'
+version = '1.0-SNAPSHOT'
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'org.apache.hadoop:hadoop-common:3.2.1'
+    implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:3.2.1'
+    implementation 'org.apache.hadoop:hadoop-mapreduce-client-jobclient:3.2.1'
+
+    testImplementation platform('org.junit:junit-bom:5.10.0')
+    testImplementation 'org.junit.jupiter:junit-jupiter'
+    testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+}
+
+java {
+    toolchain {
+        languageVersion = JavaLanguageVersion.of(8)
+    }
+}
+
+tasks.withType(JavaCompile).configureEach {
+    sourceCompatibility = JavaVersion.VERSION_1_8
+    targetCompatibility = JavaVersion.VERSION_1_8
+    options.encoding = 'UTF-8'
+}
+
+jar {
+    manifest {
+        attributes(
+                'Main-Class': 'ru.taymir.telecom.sales.Driver'
+        )
+    }
+}
+
+shadowJar {
+    archiveClassifier.set('all')
+    mergeServiceFiles()
+}
+
+test {
+    useJUnitPlatform()
+}

docker-compose.yml

@@ -0,0 +1,76 @@
+services:
+  namenode:
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
+    container_name: hadoop-namenode
+    restart: unless-stopped
+    env_file:
+      - ./hadoop/hadoop.env
+    environment:
+      - CLUSTER_NAME=${HADOOP_CLUSTER_NAME:-lab3-hadoop}
+    ports:
+      - "9870:9870"
+      - "9000:9000"
+    volumes:
+      - namenode:/hadoop/dfs/name
+
+  datanode:
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
+    container_name: hadoop-datanode
+    restart: unless-stopped
+    env_file:
+      - ./hadoop/hadoop.env
+    environment:
+      - SERVICE_PRECONDITION=namenode:9870
+    ports:
+      - "9864:9864"
+    volumes:
+      - datanode:/hadoop/dfs/data
+    depends_on:
+      - namenode
+
+  resourcemanager:
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
+    container_name: hadoop-resourcemanager
+    restart: unless-stopped
+    env_file:
+      - ./hadoop/hadoop.env
+    environment:
+      - SERVICE_PRECONDITION=namenode:9000 namenode:9870 datanode:9864
+    ports:
+      - "8088:8088"
+    depends_on:
+      - namenode
+      - datanode
+
+  nodemanager:
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
+    container_name: hadoop-nodemanager
+    restart: unless-stopped
+    env_file:
+      - ./hadoop/hadoop.env
+    environment:
+      - SERVICE_PRECONDITION=namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088
+    ports:
+      - "8042:8042"
+    depends_on:
+      - resourcemanager
+
+  historyserver:
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
+    container_name: hadoop-historyserver
+    restart: unless-stopped
+    env_file:
+      - ./hadoop/hadoop.env
+    environment:
+      - SERVICE_PRECONDITION=namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088
+    ports:
+      - "8188:8188"
+    volumes:
+      - historyserver:/hadoop/yarn/timeline
+    depends_on:
+      - resourcemanager
+
+volumes:
+  namenode:
+  datanode:
+  historyserver:

gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Sun Dec 14 22:42:04 MSK 2025
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists