Skip to content

Commit bb9618d

Browse files
iVishalriVishalr
committed
Works. Need to test out edge cases
1 parent f6137d4 commit bb9618d

File tree

8 files changed

+656788
-33
lines changed

8 files changed

+656788
-33
lines changed

‎answer/A3T1/part-00000

+2,896
Large diffs are not rendered by default.

‎docker/A3/T1/spark-input.csv

+653,674
Large diffs are not rendered by default.

‎docker/A3/T2/kafka-input.csv

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
Andaman and Nicobar,South Andaman,Port Blair,Amaranthus,Amaranthus.1,04/03/2019,6000,8000,7000
2+
Uttar Pradesh,Agra,Samsabad,Gur(Jaggery),Pathari,04/03/2019,4900.0,5100.0,5000.0
3+
Telangana,Hyderabad,Mahboob Manison,Dry Chillies,1st Sort,04/03/2019,6000.0,8500.0,7000.0
4+
Punjab,Mansa,Mansa,Bhindi(Ladies Finger),Bhindi,04/03/2019,3000.0,4000.0,3500.0
5+
Kerala,Kottayam,Ettumanoor,Cabbage,Other,04/03/2019,2000.0,2200.0,2100.0
6+
Uttar Pradesh,Agra,Khairagarh,Mustard,Sarson(Black),04/03/2019,3300.0,3500.0,3400.0
7+
Jammu and Kashmir,Badgam,Zaloosa-Chararishrief (F&V),Cucumbar(Kheera),Cucumbar,04/03/2019,1500.0,1700.0,1600.0
8+
Punjab,Muktsar,Malout,Spinach,Spinach,04/03/2019,500.0,800.0,600.0
9+
Gujarat,Vadodara(Baroda),Padra,Tomato,Other,04/03/2019,1200.0,1500.0,1350.0
10+
Rajasthan,Jalore,Jalore,Lemon,Other,04/03/2019,2300.0,2500.0,2400.0
11+
Gujarat,Kachchh,Bachau,Guar Seed(Cluster Beans Seed),Whole,04/03/2019,4000.0,4025.0,4010.0
12+
Uttar Pradesh,Kannuj,Chhibramau(Kannuj),Mustard Oil,Mustard Oil,04/03/2019,8900.0,9090.0,9000.0
13+
Kerala,Kasargod,Manjeswaram,Little gourd (Kundru),Other,04/03/2019,4200.0,5000.0,4600.0
14+
Kerala,Kollam,Anchal,Banana,Red Banana,04/03/2019,4700.0,4900.0,4800.0
15+
Gujarat,Vadodara(Baroda),Padra,Sweet Potato,Other,04/03/2019,1500.0,2000.0,1700.0
16+
Gujarat,Panchmahals,Gogamba,Paddy(Dhan)(Common),G. R. 11,04/03/2019,1500.0,1550.0,1525.0
17+
Uttar Pradesh,Aligarh,Atrauli,Maize,Hybrid,04/03/2019,1930.0,1950.0,1940.0
18+
Odisha,Mayurbhanja,Betnoti,Leafy Vegetable,Other,04/03/2019,1400.0,1600.0,1500.0
19+
Tamil Nadu,Coimbatore,Annur,Tobacco,Other,04/03/2019,3000.0,3200.0,3100.0
20+
Uttar Pradesh,Bhadohi(Sant Ravi Nagar),Gopiganj,Potato,Potato,04/03/2019,350.0,400.0,400.0
21+
Maharashtra,Pune,Pune,Cucumbar(Kheera),Other,04/03/2019,1000.0,3000.0,2000.0
22+
Tamil Nadu,Coimbatore,Palladam,Coconut,Other,04/03/2019,2040.0,2240.0,2140.0
23+
West Bengal,Burdwan,Burdwan,Sweet Pumpkin,Sweet Pumpkin,04/03/2019,950.0,1050.0,1000.0
24+
Manipur,Bishnupur,Bishenpur,Paddy(Dhan)(Common),Other,04/03/2019,2250.0,2250.0,2250.0
25+
Uttar Pradesh,Saharanpur,Sultanpurchilkana,Carrot,Carrot,04/03/2019,400.0,500.0,450.0
26+
Kerala,Thirssur,Kodungalloor,Tomato,Other,04/03/2019,1600.0,1600.0,1600.0
27+
Assam,Kamrup,Pamohi(Garchuk),Pumpkin,Pumpkin,04/03/2019,800.0,1000.0,900.0
28+
Uttar Pradesh,Agra,Samsabad,Barley (Jau),Dara,04/03/2019,1825.0,1875.0,1850.0
29+
Punjab,Muktsar,Malout,Green Chilli,Green Chilly,04/03/2019,800.0,3500.0,2000.0
30+
Uttar Pradesh,Muzaffarnagar,Muzzafarnagar,Gur(Jaggery),Yellow,04/03/2019,2400.0,2600.0,2510.0
31+
Punjab,Hoshiarpur,Dasuya,Kinnow,Other,04/03/2019,1600.0,2000.0,2000.0
32+
Rajasthan,Jalore,Jalore,Pomegranate,Other,04/03/2019,1400.0,1600.0,1500.0
33+
Uttar Pradesh,Baghpat,Bagpat,Carrot,Carrot,04/03/2019,400.0,600.0,500.0
34+
West Bengal,Bankura,Bishnupur(Bankura),Rice,Fine,04/03/2019,2800.0,2900.0,2850.0
35+
Kerala,Alappuzha,Mannar,Carrot,Other,04/03/2019,3700.0,3900.0,3800.0
36+
Odisha,Sundergarh,Bonai,Potato,Other,04/03/2019,1000.0,1000.0,1000.0
37+
Tamil Nadu,Villupuram,Manalurpet,Ragi (Finger Millet),Other,04/03/2019,2312.0,2312.0,2312.0
38+
Uttar Pradesh,Bahraich,Ruperdeeha,Grapes,Green,04/03/2019,4480.0,4540.0,4500.0
39+
Maharashtra,Satara,Karad,Cabbage,Other,04/03/2019,500.0,1000.0,1000.0
40+
Tamil Nadu,Cuddalore,Cuddalore,Paddy(Dhan)(Common),B P T,04/03/2019,1344.0,1555.0,1500.0
41+
Kerala,Kottayam,Ettumanoor,Onion,Big,04/03/2019,3300.0,3500.0,3500.0
42+
West Bengal,Medinipur(W),Ghatal,"Sesamum(Sesame,Gingelly,Til)",Red,04/03/2019,8500.0,8700.0,8600.0
43+
Rajasthan,Jalore,Jalore,Bottle gourd,Other,04/03/2019,1000.0,1200.0,1100.0
44+
Uttar Pradesh,Baghpat,Baraut,Raddish,Raddish,04/03/2019,400.0,600.0,500.0
45+
Maharashtra,Buldhana,Deoulgaon Raja,Cotton,Desi,04/03/2019,5100.0,5400.0,5300.0
46+
Kerala,Thirssur,Chelakkara,Elephant Yam (Suran),Other,04/03/2019,2500.0,2800.0,2600.0
47+
Tamil Nadu,Salem,Salem,Turmeric,Finger,04/03/2019,6940.0,7395.0,7120.0
48+
Odisha,Ganjam,Digapahandi,Maize,Other,04/03/2019,1300.0,1420.0,1350.0
49+
Gujarat,Vadodara(Baroda),Padra,Brinjal,Other,04/03/2019,400.0,1000.0,750.0
50+
Uttar Pradesh,Agra,Achnera,Barley (Jau),Dara,04/03/2019,1800.0,1860.0,1830.0
51+
Uttar Pradesh,Muzaffarnagar,Muzzafarnagar,Green Chilli,Green Chilly,04/03/2019,2230.0,2300.0,2260.0
52+
Haryana,Gurgaon,Pataudi,Spinach,Other,04/03/2019,500.0,1000.0,750.0
53+
Kerala,Malappuram,Kondotty,Tomato,Other,04/03/2019,1000.0,1200.0,1100.0
54+
Odisha,Balasore,Jaleswar,Onion,Other,04/03/2019,1000.0,1150.0,1100.0
55+
Tamil Nadu,Villupuram,Vikkiravandi,Hybrid Cumbu,Other,04/03/2019,1800.0,1853.0,1825.0
56+
Kerala,Malappuram,Kondotty,Cowpea(Veg),Other,04/03/2019,3000.0,3200.0,3100.0
57+
Kerala,Malappuram,Kondotty,Lemon,Other,04/03/2019,4000.0,4200.0,4100.0
58+
Punjab,Tarntaran,Patti,Spinach,Other,04/03/2019,300.0,430.0,400.0
59+
Uttar Pradesh,Saharanpur,Nanuta,Green Chilli,Green Chilly,04/03/2019,1200.0,1300.0,1250.0
60+
Manipur,Imphal East,Lamlong Bazaar,Onion,Other,04/03/2019,1300.0,2500.0,2500.0
61+
Tripura,North Tripura,Dasda,Rice,Masuri,04/03/2019,2500.0,2700.0,2600.0
62+
Rajasthan,Kota,Itawa,Wheat,Other,04/03/2019,1915.0,1915.0,1915.0
63+
Uttar Pradesh,Sonbhadra,Robertsganj,Garlic,Garlic,04/03/2019,1750.0,1840.0,1810.0
64+
Odisha,Dhenkanal,Hindol,Brinjal,Brinjal,04/03/2019,800.0,1200.0,1000.0
65+
Uttar Pradesh,Bulandshahar,Divai,Pumpkin,Pumpkin,04/03/2019,680.0,700.0,690.0
66+
Meghalaya,East Khasi Hills,Shillong,Ginger(Dry),Dry,04/03/2019,6000.0,7000.0,6500.0
67+
Uttar Pradesh,Kannuj,Chhibramau(Kannuj),Cauliflower,Local,04/03/2019,450.0,500.0,480.0
68+
Madhya Pradesh,Dindori,Gorakhpur,Paddy(Dhan)(Common),Other,04/03/2019,1300.0,1350.0,1325.0
69+
Haryana,Mewat,Taura,Potato,Potato,04/03/2019,700.0,700.0,700.0
70+
Manipur,Bishnupur,Bishenpur,Cabbage,Cabbage,04/03/2019,1100.0,1300.0,1300.0
71+
Uttar Pradesh,Muzaffarnagar,Muzzafarnagar,Pumpkin,Pumpkin,04/03/2019,750.0,830.0,800.0
72+
Uttar Pradesh,Kannuj,Chhibramau(Kannuj),Mustard,Lohi Black,04/03/2019,3300.0,3450.0,3400.0
73+
Maharashtra,Satara,Karad,Carrot,Other,04/03/2019,500.0,1000.0,1000.0
74+
Uttar Pradesh,Saharanpur,Gangoh,Tomato,Deshi,04/03/2019,1325.0,1350.0,1340.0
75+
Kerala,Kasargod,Manjeswaram,Bottle gourd,Bottle Gourd,04/03/2019,1600.0,2000.0,1800.0
76+
Uttar Pradesh,Muzaffarnagar,Thanabhawan,Cauliflower,Local,04/03/2019,300.0,400.0,340.0
77+
Uttar Pradesh,Agra,Jarar,Green Chilli,Green Chilly,04/03/2019,1040.0,1120.0,1080.0
78+
Himachal Pradesh,Kangra,Kangra(Jassour),Methi(Leaves),Other,04/03/2019,900.0,1000.0,950.0
79+
Maharashtra,Satara,Karad,Spinach,Other,04/03/2019,1.0,2.0,2.0
80+
Gujarat,Bharuch,Ankleshwar,Methi(Leaves),Methi,04/03/2019,600.0,1000.0,825.0
81+
Kerala,Thiruvananthapuram,Maranelloor,Banana,Poovan,04/03/2019,2900.0,3200.0,3000.0
82+
Tamil Nadu,Villupuram,Villupuram,Black Gram (Urd Beans)(Whole),Other,04/03/2019,2559.0,2639.0,2619.0
83+
West Bengal,Puruliya,Balarampur,Tomato,Other,04/03/2019,250.0,350.0,300.0
84+
Kerala,Thiruvananthapuram,Parassala,Snakeguard,Snakeguard,04/03/2019,2600.0,3000.0,2600.0
85+
Kerala,Thirssur,Irinjalakkuda,Bitter gourd,Other,04/03/2019,4000.0,4000.0,4000.0
86+
Uttar Pradesh,Ghaziabad,Hapur,Wheat,Dara,04/03/2019,1900.0,2100.0,1980.0
87+
Punjab,Jalandhar,Mehatpur,Spinach,Other,04/03/2019,400.0,600.0,600.0
88+
Himachal Pradesh,Kangra,Kangra(Jassour),Pomegranate,Other,04/03/2019,4000.0,4500.0,4250.0
89+
Assam,Kamrup,Pamohi(Garchuk),Brinjal,Brinjal,04/03/2019,1200.0,1400.0,1300.0
90+
Punjab,Muktsar,Malout,Peas cod,Other,04/03/2019,1000.0,1500.0,1200.0
91+
Kerala,Thiruvananthapuram,Parassala,Bitter gourd,Bitter Gourd,04/03/2019,5000.0,5500.0,5000.0
92+
Rajasthan,Jalore,Jalore,Bhindi(Ladies Finger),Other,04/03/2019,3800.0,4000.0,3900.0
93+
Kerala,Thirssur,Irinjalakkuda,Elephant Yam (Suran),Other,04/03/2019,2400.0,2400.0,2400.0
94+
Uttar Pradesh,Agra,Fatehabad,Barley (Jau),Dara,04/03/2019,1700.0,1750.0,1720.0
95+
Kerala,Alappuzha,Chengannur,Ashgourd,Other,04/03/2019,1100.0,1400.0,1200.0
96+
Tamil Nadu,Villupuram,Tindivanam,Groundnut,Other,04/03/2019,6236.0,7387.0,6724.0
97+
Kerala,Malappuram,Kondotty,Potato,Other,04/03/2019,2000.0,2200.0,2100.0
98+
Uttar Pradesh,Baghpat,Bagpat,Tomato,Deshi,04/03/2019,1100.0,1300.0,1200.0
99+
Tripura,South District,Barpathari,Potato,Jalander,04/03/2019,700.0,800.0,750.0
100+
West Bengal,Puruliya,Purulia,Moath Dal,Moath Dal,04/03/2019,7300.0,7400.0,7400.0
101+
Maharashtra,Kolhapur,Kolhapur,Brinjal,Other,04/03/2019,600.0,3000.0,1800.0
102+
Punjab,Ludhiana,Sahnewal,Methi(Leaves),Methi,04/03/2019,1000.0,1000.0,1000.0
103+
Punjab,Muktsar,Malout,Guava,Guava,04/03/2019,2000.0,2500.0,2200.0
104+
Tamil Nadu,Erode,Dharapuram,Coconut,Big,04/03/2019,2505.0,3275.0,3000.0
105+
Maharashtra,Nagpur,Savner,Wheat,Other,04/03/2019,1935.0,2040.0,2000.0
106+
Punjab,Jalandhar,Adampur,Amphophalus,Other,04/03/2019,1800.0,2000.0,2000.0
107+
Uttar Pradesh,Lakhimpur,Paliakala,Bottle gourd,Bottle Gourd,04/03/2019,1020.0,1080.0,1050.0
108+
Uttar Pradesh,Muzaffarnagar,Thanabhawan,Brinjal,Brinjal,04/03/2019,400.0,500.0,440.0
109+
West Bengal,Nadia,Nadia,Mustard,Yellow (Black),04/03/2019,4000.0,4150.0,4100.0
110+
Gujarat,Bharuch,Ankleshwar,Green Chilli,Green Chilly,04/03/2019,2600.0,3300.0,3000.0
111+
Haryana,Ambala,Shahzadpur,Ber(Zizyphus/Borehannu),Ber(Zizyphus),04/03/2019,2000.0,4000.0,3000.0
112+
Uttar Pradesh,Agra,Fatehabad,Onion,Red,04/03/2019,850.0,1000.0,900.0
113+
Tamil Nadu,Namakkal,Namagiripettai,Turmeric,Bulb,04/03/2019,5050.0,6050.0,5740.0
114+
EOF

‎docker/hadoop3-A3.dockerfile

+2
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ RUN mv page_embeddings.json /A2/
7979
RUN mv w /A2/
8080
RUN mv adjacency_list.txt /A2/
8181

82+
ADD A3 /A3
83+
8284
# add new ports here
8385
EXPOSE 8088 50070 50075 50030 50060 9870 10000 19888
8486
# RUN swapoff -a

‎docker/hadoop_server-A3.py

+84-25
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131

3232
JOBHISTORY_URL = "http://localhost:19888/ws/v1/history/mapreduce/jobs"
3333

34-
TASK_OUTPUT_PATH = {"A3T1":"task-1-output", "A3T2":"task-2-output"}
34+
TASK_OUTPUT_PATH = {"A3T1":"task-1-output", "A3T2":"task-2-output", "A2T1":"task-1-output", "A2T2":"task-2-output"}
3535

3636
FILEPATH = os.path.join(os.getcwd(), 'output')
3737

@@ -208,6 +208,15 @@ def run_spark_job(team_id, assignment_id, submission_id, timeout, spark:str):
208208
st = os.stat(os.path.join(task_path,"spark.py"))
209209
os.chmod(os.path.join(task_path,"spark.py"), st.st_mode | stat.S_IEXEC)
210210

211+
res = create_hdfs_directory(f"/{team_id}")
212+
if res != 0:
213+
print(f"Failed to create HDFS Directory : hdfs:/{team_id}")
214+
215+
directory = f"/{team_id}/{assignment_id}"
216+
res = create_hdfs_directory(directory)
217+
if res != 0:
218+
print(f"Failed to create HDFS Directory : hdfs:{directory}")
219+
211220
task_path = os.path.join(path, submission_id)
212221

213222
timestamp = str(time.time())
@@ -221,7 +230,7 @@ def run_spark_job(team_id, assignment_id, submission_id, timeout, spark:str):
221230
if assignment_id == "A3T1":
222231
logger.mark(f"Spawning Spark Process")
223232

224-
spark_job = f'''{SPARK} --name={job_name} --master=yarn --deploy-mode=cluster "/{os.path.join(task_path,'spark.py')}" "/A3/spark-input.csv" "/{os.path.join(task_path,'spark-output')}"'''
233+
spark_job = f'''{SPARK} --name={job_name} --master=yarn --deploy-mode=cluster /{os.path.join(task_path,'spark.py')} /A3/T1/spark-input.csv /{team_id}/{assignment_id}/{TASK_OUTPUT_PATH[assignment_id]}'''
225234

226235
spark_process = subprocess.Popen([
227236
spark_job
@@ -240,7 +249,7 @@ def run_spark_job(team_id, assignment_id, submission_id, timeout, spark:str):
240249
flag = False
241250
else:
242251
current_job = data[0]
243-
current_job_state = current_job["state"]
252+
current_job_state = current_job["finalStatus"]
244253
application_id = current_job["id"]
245254

246255
if not os.path.exists(os.path.join(FILEPATH, team_id, assignment_id)):
@@ -252,12 +261,12 @@ def run_spark_job(team_id, assignment_id, submission_id, timeout, spark:str):
252261
job_output = "Good Job!"
253262
status = current_job["state"]
254263

255-
if os.path.exists(os.path.join(FILEPATH, team_id, assignment_id, "part-*")):
256-
os.remove(os.path.join(FILEPATH, team_id, assignment_id, "part-*"))
264+
if os.path.exists(os.path.join(FILEPATH, team_id, assignment_id, "part-00000")):
265+
os.remove(os.path.join(FILEPATH, team_id, assignment_id, "part-00000"))
257266
# os.remove(os.path.join(FILEPATH, team_id, assignment_id, "_SUCCESS"))
258-
259-
process = subprocess.Popen([f"cp -R /{os.join(task_path, 'spark-output', 'part-*')} /{os.path.join(FILEPATH, team_id, assignment_id)}"], shell=True, text=True)
260-
_ = process.wait()
267+
268+
process = subprocess.Popen([f"{HDFS} dfs -get /{team_id}/{assignment_id}/{TASK_OUTPUT_PATH[assignment_id]}/part-* {os.path.join(FILEPATH, team_id, assignment_id)}"], shell=True, text=True)
269+
process_code = process.wait()
261270

262271
process = subprocess.Popen([f"mv /{os.path.join(FILEPATH, team_id, assignment_id, 'part-*')} /{os.path.join(FILEPATH, team_id, assignment_id, 'part-00000')}"], shell=True, text=True)
263272
_ = process.wait()
@@ -362,41 +371,85 @@ def run_kafka_job(team_id, assignment_id, submission_id, timeout, producer:str,
362371
if os.path.exists(os.path.join(FILEPATH, team_id, assignment_id, "output.json")):
363372
os.remove(os.path.join(FILEPATH, team_id, assignment_id, "output.json"))
364373

365-
consumer_cmd = f'''python3 /{os.path.join(task_path, "consumer.py")} agriculture > /{os.path.join(FILEPATH, team_id, assignment_id, "output.json")}'''
366-
producer_cmd = f'''cat /A3/dataset.csv | python3 /{os.path.join(task_path, "producer.py")} agriculture'''
374+
consumer_cmd = f'''python3 /{os.path.join(task_path, "consumer.py")} "agriculture" > /{os.path.join(FILEPATH, team_id, assignment_id, "output.json")}'''
375+
producer_cmd = f'''cat /A3/T2/kafka-input.csv | python3 /{os.path.join(task_path, "producer.py")} "agriculture"'''
367376

377+
print("Starting Consumer Process")
368378
consumer_process = subprocess.Popen([
369379
consumer_cmd
370380
], shell=True, text=True, preexec_fn=os.setsid, stdout=subprocess.PIPE)
371381

382+
time.sleep(5)
383+
384+
print("Starting Producer Process")
372385
producer_process = subprocess.Popen([
373386
producer_cmd
374387
], shell=True, text=True, preexec_fn=os.setsid, stdout=subprocess.PIPE)
375388

376-
kafka_process_code1 = producer_process.wait(timeout=timeout)
377-
kafka_process_code2 = consumer_process.wait(timeout=timeout)
389+
flag = True
378390

379-
if os.path.exists(os.path.join(FILEPATH, team_id, assignment_id, "output.json")):
391+
error_logs = [
392+
f"Submission ID : {submission_id}",
393+
f"Team ID : {team_id}",
394+
f"Assignment ID : {assignment_id}",
395+
"Note : If you do not see any error with respect to your code and you only see : \n\nlog4j:WARN\n\nThen that means your code had infinite loop and submission was killed.\n\nLOGS :- \n\n",
396+
]
397+
398+
consumer_stderr = consumer_process.communicate()[1]
399+
if consumer_stderr:
400+
error_logs.append(
401+
"consumer\n--------\n\n"+consumer_stderr
402+
)
403+
consumer_process.kill()
404+
405+
try:
406+
kafka_process_code2 = consumer_process.wait(timeout=(timeout-10)//2)
407+
print("Consumer Completed!")
408+
except subprocess.TimeoutExpired:
409+
logger.mark(f"Team ID : {team_id} Assignment ID : {assignment_id} Kafka Job Failed")
410+
msg = f"Team ID : {team_id} Assignment ID : {assignment_id} Kafka Job Failed. Time Limit Exceeded in Consumer."
411+
status = "FAILED"
412+
job_output = "Failed! Time Limit Exceeded in Consumer."
413+
flag = False
414+
415+
producer_stderr = producer_process.communicate()[1]
416+
if producer_stderr:
417+
error_logs.append(
418+
"producer\n--------\n\n"+producer_stderr
419+
)
420+
producer_process.kill()
421+
422+
try:
423+
kafka_process_code1 = producer_process.wait(timeout=(timeout-10)//2)
424+
print("Producer Completed!")
425+
except subprocess.TimeoutExpired:
426+
logger.mark(f"Team ID : {team_id} Assignment ID : {assignment_id} Kafka Job Failed")
427+
msg = f"Team ID : {team_id} Assignment ID : {assignment_id} Kafka Job Failed. Time Limit Exceeded in Producer."
428+
status = "FAILED"
429+
job_output = "Failed! Time Limit Exceeded in Producer."
430+
flag = False
431+
432+
if flag and os.path.exists(os.path.join(FILEPATH, team_id, assignment_id, "output.json")):
380433
if os.stat(os.path.join(FILEPATH, team_id, assignment_id, "output.json")).st_size == 0 or (kafka_process_code1 != 0 or kafka_process_code2 != 0):
381434
logger.mark(f"Team ID : {team_id} Assignment ID : {assignment_id} Kafka Job Failed")
382435
msg = f"Team ID : {team_id} Assignment ID : {assignment_id} Kafka Job Failed."
383436
status = "FAILED"
384437
job_output = "Failed! Your submission might have thrown error(s). Logs have been mailed to you."
385438

386-
error_logs = [
387-
f"Submission ID : {submission_id}",
388-
f"Team ID : {team_id}",
389-
f"Assignment ID : {assignment_id}",
390-
"Note : If you do not see any error with respect to your code and you only see : \n\nlog4j:WARN\n\nThen that means your code had infinite loop and submission was killed.\n\nLOGS :- \n\n",
391-
]
439+
# error_logs = [
440+
# f"Submission ID : {submission_id}",
441+
# f"Team ID : {team_id}",
442+
# f"Assignment ID : {assignment_id}",
443+
# "Note : If you do not see any error with respect to your code and you only see : \n\nlog4j:WARN\n\nThen that means your code had infinite loop and submission was killed.\n\nLOGS :- \n\n",
444+
# ]
392445

393-
error_logs.append(
394-
"producer\n--------\n\n"+producer_process.communicate()[1]
395-
)
446+
# error_logs.append(
447+
# "producer\n--------\n\n"+producer_process.communicate()[1]
448+
# )
396449

397-
error_logs.append(
398-
"consumer\n--------\n\n"+consumer_process.communicate()[1]
399-
)
450+
# error_logs.append(
451+
# "consumer\n--------\n\n"+consumer_process.communicate()[1]
452+
# )
400453

401454
with open(os.path.join(FILEPATH, team_id, assignment_id, "error.txt"), "w+") as f:
402455
f.write(error_logs)
@@ -800,11 +853,17 @@ def initialize_environment(add_dataset=True) -> Tuple:
800853
_ = create_hdfs_directory("/A2")
801854
_ = create_hdfs_directory("/A2/input")
802855

856+
_ = create_hdfs_directory("/A3")
857+
_ = create_hdfs_directory("/A3/T1")
858+
803859
p = subprocess.Popen([f"{HDFS} dfs -put /A2/graph.txt /A2/input"], shell=True, text=True)
804860
_ = p.wait()
805861

806862
p = subprocess.Popen([f"{HDFS} dfs -put /A2/adjacency_list.txt /A2/input"], shell=True, text=True)
807863
_ = p.wait()
864+
865+
p = subprocess.Popen([f"{HDFS} dfs -put /A3/T1/spark-input.csv /A3/T1/"], shell=True, text=True)
866+
_ = p.wait()
808867

809868
return 0, error_logs
810869

‎flask_backend/app.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ def sanity_check():
319319
consumer.write(consumer_data)
320320
consumer.close()
321321

322-
process = subprocess.Popen([f'pylint --disable=I,R,C,W {os.path.join(os.getcwd(), "compile-test/")}'], shell=True, stdout=subprocess.PIPE, text=True)
322+
process = subprocess.Popen([f'pylint --disable=R,C,W,import-error {os.path.join(os.getcwd(), "compile-test/")}'], shell=True, stdout=subprocess.PIPE, text=True)
323323
exit_code = process.wait()
324324

325325
output = process.communicate()[0]

‎job_tracker/executor.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -313,13 +313,13 @@ def cleanup(self):
313313
fetch_ip=BACKEND_INTERNAL_IP,
314314
fetch_port=9000,
315315
fetch_route="get-jobs",
316-
num_workers=4,
316+
num_workers=1,
317317
global_queue_thread=True,
318318
global_prefetch_thread=True,
319319
prefetch_threads=4,
320320
prefetch_factor=4,
321321
threshold=10,
322-
num_backends=8
322+
num_backends=1
323323
)
324324

325325
print(executor)
@@ -329,8 +329,8 @@ def cleanup(self):
329329
docker_route = "run_job"
330330
docker_image = "hadoop-3.2.2:0.1"
331331

332-
backend_cpu_limit: int = 3
333-
backend_mem_limit: str = "12000m"
332+
backend_cpu_limit: int = 6
333+
backend_mem_limit: str = "8000m"
334334
backend_host_output_dir: str = f"{os.path.join(os.getcwd(),'output')}"
335335
backend_docker_output_dir: str = f"/output"
336336
backend_memswapiness: int = 0

0 commit comments

Comments
 (0)