Python looks excruciatingly slow to me. If you want fast I believe you need to think and write in vector languages like kdb+/q. I am not a kdb+ expert by any means and my code can probably use more q primitives, but here was my solution in ~2 minutes:
i1:("I I";" ")0: `:1.txt;
sum {abs last deltas x }each flip asc each i1 / answer 1
sum {x * sum x = i1[1]}each i1[0] / answer 2
from collections import *
xys = list(map(int, open(0).read().split()))
xs = xys[::2]
ys = xys[1::2]
xs.sort()
ys.sort()
print(sum(abs(x-y) for x,y in zip(xs,ys)))
yc = Counter(ys)
print(sum(((yc[x])*x for x in xs)))
data = { i+1 : sorted([ x for x in list(map(int, open('input').read().split()))[i::2]]) for i in range(2) }
total_distance = sum(list(map(lambda x: abs(x[0]-x[1]), zip(data[1], data[2]))))
print("part 1:", total_distance)
similarity_score = sum(list(map(lambda x: (x*data[2].count(x))*data[1].count(x), set(data[1]).intersection(data[2]))))
print("part 2:", similarity_score)
