Skip to content

Commit

Permalink
make sure structure crawling does not crawl elements
Browse files Browse the repository at this point in the history 
session details are written to file, printed to console and deleted after successful login
  • Loading branch information
@abelhegedus
abelhegedus committed Nov 25, 2019
1 parent f16a17a commit a11dd4c
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 72 deletions.
7 changes: 7 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@

Running the jar without arguments prints out the usage instructions.

- Running the crawler without workspace, resource, branch and revision will result in crawling the repository structure of the given scope
- Running the crawler with workspace, resource, branch and revision will result in crawling the model content of the given revision

### Repository structure of OpenSE Cookbook workspace on twc.openmbee.org

`-S twc.openmbee.org -P 8111 -ssl -W 9c368adc-10cc-45d9-bec6-27aedc80e68b -C 2000 -u openmbeeguest -pw guest`

### OpenSE Cookbook model on twc.openmbee.org

`-S twc.openmbee.org -P 8111 -ssl -W 9c368adc-10cc-45d9-bec6-27aedc80e68b -R c6bede89-cd5e-487b-aca8-a4f384370759 -B 29110c0f-bdc1-4294-ae24-9fd608629cac -REV 350 -C 2000 -u openmbeeguest -pw guest`
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.3.50'
id 'org.jetbrains.kotlin.jvm' version '1.3.60'
id 'io.vertx.vertx-plugin' version '0.9.0'
}

Expand Down
19 changes: 8 additions & 11 deletions src/main/kotlin/Crawler.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ import io.vertx.core.json.Json
import io.vertx.core.json.JsonObject
import java.io.File


private val CHUNK_SIZE = "chunkSize"
const val CHUNK_SIZE = "chunkSize"

fun main(args: Array<String>) {

Expand Down Expand Up @@ -128,11 +127,9 @@ fun main(args: Array<String>) {
twcMap[CHUNK_SIZE] = chunkSize
println("Chunk size is $chunkSize")

if (instanceNum != null) {
println("Instance number set to $instanceNum")
if (instanceNum < 1) {
error("Number of Instances should be at least 1.")
}
println("Instance number set to $instanceNum")
if (instanceNum < 1) {
error("Number of Instances should be at least 1.")
}
if (workspaceId != null) {
println("Workspace ID set to $workspaceId")
Expand Down Expand Up @@ -200,7 +197,7 @@ fun main(args: Array<String>) {

val restVerticle = RESTVerticle()

var options = DeploymentOptions().setWorker(true).setHa(true).setInstances(instanceNum).setWorkerPoolSize(32)
val options = DeploymentOptions().setWorker(true).setHa(true).setInstances(instanceNum).setWorkerPoolSize(32)

twcMap["flag"] = 0
vertx.deployVerticle(restVerticle.javaClass.name, options) { deploy ->
Expand All @@ -217,9 +214,9 @@ fun main(args: Array<String>) {
usr,
pswd
)
) { deploy ->
if (deploy.failed()) {
error("Deploy failed: ${deploy.cause().message}\n${deploy.cause().printStackTrace()}")
) {
if (it.failed()) {
error("Deploy failed: ${it.cause().message}\n${it.cause().printStackTrace()}")
}
}
}
Expand Down
117 changes: 58 additions & 59 deletions src/main/kotlin/verticles/MainVerticle.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,29 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {

val eb = vertx.eventBus()
vertx.setPeriodic(1000) {
vertx.sharedData().getCounter(QUERIES) { res ->
if (res.succeeded()) {
val counter = res.result()
counter.get { res ->
queries = res.result().toInt()
vertx.sharedData().getCounter(QUERIES) { queriesR ->
if (queriesR.succeeded()) {
val queriesC = queriesR.result()
queriesC.get { queriesV ->
this.queries = queriesV.result().toInt()

vertx.sharedData().getCounter("sum") { res ->
if (res.succeeded()) {
val counter = res.result()
counter.get { get ->
if (get.succeeded()) {
sum = get.result().toInt()
vertx.sharedData().getCounter("sum") { sumR ->
if (sumR.succeeded()) {
val sumC = sumR.result()
sumC.get { sumV ->
if (sumV.succeeded()) {
this.sum = sumV.result().toInt()

vertx.sharedData().getCounter("number") { res ->
if (res.succeeded()) {
val counter = res.result()
counter.get { get ->
if (get.succeeded()) {
number = get.result().toInt()
println("Total: ${queries}/${sum}/${++s} query/response/sec | Now: $number elem | AvgSpeed: ${sum / (s)} elem/sec")
counter.compareAndSet(number.toLong(), 0, {})
vertx.sharedData().getCounter("number") { numR ->
if (numR.succeeded()) {
val numC = numR.result()
numC.get { numV ->
if (numV.succeeded()) {
number = numV.result().toInt()
println("Total: ${this.queries}/${this.sum}/${++s} query/response/sec | Now: $number elem | AvgSpeed: ${this.sum / (s)} elem/sec")
numC.compareAndSet(number.toLong(), 0, {})

if (queries != 0 && sum != 0 && queries == sum) {
if (this.queries != 0 && this.sum != 0 && this.queries == this.sum) {
eb.send(
TWCVERT_ADDRESS,
JsonObject.mapFrom(
Expand All @@ -52,13 +52,13 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
}
}
} else {
error("Counter: queries not available.")
error("Counter: queriesR not available.")
}
}
}
}
} else {
error("Counter: queries not available.")
error("Counter: queriesR not available.")
}
}

Expand Down Expand Up @@ -175,8 +175,8 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {

}
REPO -> {
// println("Received Repository")
val repo = JsonObject(messageData.obj as Map<String, Any>).mapTo(Repo::class.java)
println("Received workspace list $repo")
repo.workspaces.forEach { ws ->
val id = JsonObject(ws as Map<String, Any>).getString("@id")
eb.send(
Expand All @@ -194,8 +194,8 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
}
}
WORKSPACE -> {
// println("Received Workspace")
val workspace = JsonObject(messageData.obj as Map<String, Any>).mapTo(Workspace::class.java)
println("Received workspace content for $workspace")
val workspaceId = workspace.id

workspace.resources.forEach { res ->
Expand All @@ -219,8 +219,8 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
}
}
RESOURCE -> {
// println("Received Resource")
val resource = JsonObject(messageData.obj as Map<String, Any>).mapTo(Resource::class.java)
println("Received resource content for $resource")
val resourceId = resource.id
val workspaceId = resource.workspace_id
resource.branches.forEach { branch ->
Expand Down Expand Up @@ -249,48 +249,47 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {

}
BRANCH -> {
// println("Received Branch")
val branch = JsonObject(messageData.obj as Map<String, Any>).mapTo(Branch::class.java)
println("Received branch content for $branch")
val branchId = branch.id
val resourceId = branch.resource_id
val workspaceId = branch.workspace_id
val inputRevision = twcMap[REVISION]
if (inputRevision == null) {
twcMap[REVISION] = branch.revisions.maxBy { it as Int }
}

branch.revisions.forEach { rev ->
val revId = (rev as Int)
vertx.eventBus().send(
TWCVERT_ADDRESS, JsonObject.mapFrom(
Message(
GET_ROOT_ELEMENT_IDS,
JsonObject()
.put(
WORKSPACE_ID,
workspaceId
)
.put(
RESOURCE_ID,
resourceId
)
.put(
BRANCH_ID,
branchId
)
.put(
REVISION_ID,
revId
)
val inputRevision = twcMap[REVISION]
if (revId == inputRevision) {
vertx.eventBus().send(
TWCVERT_ADDRESS, JsonObject.mapFrom(
Message(
GET_ROOT_ELEMENT_IDS,
JsonObject()
.put(
WORKSPACE_ID,
workspaceId
)
.put(
RESOURCE_ID,
resourceId
)
.put(
BRANCH_ID,
branchId
)
.put(
REVISION_ID,
revId
)
)
)
)
)
}
}

}
REVISION -> {
// println("Received Revision")
val revision = JsonObject(messageData.obj as Map<String, Any>).mapTo(Revision::class.java)
println("Received revision content for $revision")
val revisionId = revision.id
val branchId = revision.branch_id
val resourceId = revision.resource_id
Expand Down Expand Up @@ -365,12 +364,12 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
}
ELEMENT -> {
// println("Received Element")
val element = JsonObject(messageData.obj as Map<String, Any>).mapTo(Element::class.java)
val revisionId = element.revision_id
val branchId = element.branch_id
val resourceId = element.resource_id
val workspaceId = element.workspace_id
element.elements.forEach { element ->
val elementRequest = JsonObject(messageData.obj as Map<String, Any>).mapTo(Element::class.java)
val revisionId = elementRequest.revision_id
val branchId = elementRequest.branch_id
val resourceId = elementRequest.resource_id
val workspaceId = elementRequest.workspace_id
elementRequest.elements.forEach { element ->
val elementId = JsonObject(element as Map<String, Any>).getString("@id")

vertx.eventBus().send(
Expand Down
3 changes: 2 additions & 1 deletion src/main/kotlin/verticles/RESTVerticle.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ class RESTVerticle() : AbstractVerticle() {
}

private fun getRevisions(client: WebClient, twcMap: LocalMap<Any, Any>, obj: JsonObject) {
queryPrepared(1)
val workspaceId = obj.getString(WORKSPACE_ID)
val resourceId = obj.getString(RESOURCE_ID)
val branchId = obj.getString(BRANCH_ID)
Expand All @@ -321,7 +322,7 @@ class RESTVerticle() : AbstractVerticle() {
.send { ar ->
if (ar.succeeded()) {
if (ar.result().statusCode() == 200) {

queryCompleted(1)
val data = ar.result().bodyAsJsonArray()
val branch = Branch(
branchId,
Expand Down

0 comments on commit a11dd4c

Please sign in to comment.