Python Queue issue - queue.get doesn't remove an object

Question

I'm creating a threaded python script that puts into a queue some hosts/ips and then an unknown amount of threads (default is 10) gets this hosts from queue and executes a snmp query. Snmp query is working but every thread is doing query of all hosts. Host wants to receive only single call from random thread. Here I get the call to host from each thread.

#!/usr/bin/env python
import Queue
import threading
import urllib2
import time
import datetime
import netsnmp
import pprint

queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
    def run(self):
        while True:
      #grabs host from queue
            item = self.queue.get()
            print item

      #grabs urls of hosts and prints first 1024 bytes of page
            dane = get_rittal(item)
            obj_in_que = self.queue.qsize()
            #print obj_in_que 
            print dane + "\n"

      #signals to queue job is done
            self.queue.task_done()

def log(message):
    now = datetime.datetime.now().strftime("%H:%M:%S")
    print "%s %s\n" % (now, message)



def convert(val,div):
    string = str(val)
    f = '%05.2f' % (float(string)*div)
    return f

def get_rittal(rittal):
    x = netsnmp.Session(Version=1,DestHost=rittal,Community='',Timeout=25000, Retries=3)
    oid_loadA = netsnmp.Varbind('.1.3.6.1.4.1.2606.100.1.2.2.1.7.1',1)
    oid_loadB = netsnmp.Varbind('.1.3.6.1.4.1.2606.100.1.2.2.1.7.1',2)
    oid_loadC = netsnmp.Varbind('.1.3.6.1.4.1.2606.100.1.2.2.1.7.1',3)

    all_oids = netsnmp.VarList(oid_loadA, oid_loadB, oid_loadC)

    get = list(x.get(all_oids))

    loadA = convert(get[0],0.01)
    loadB = convert(get[1],0.01)
    loadC = convert(get[2],0.01)

    res = 'Host:' + rittal + '\t\tLA:' + loadA + 'A LB:' + loadB + 'A LC:' + loadC + 'A'
    return res

start = time.time()
def main():
    list=[]
    for line in open ("test", "r").readlines():
        for h in line.rstrip('\n').split(' '):
            #print h
            list.append(h)
    x = str(len(list))
    print "Ilosc rittali = " + x
    print list;

  #spawn a pool of threads, and pass them queue instance 
    for i in range(10):
        t = ThreadUrl(queue)
        t.setDaemon(True)
        t.start()

 #populate queue with data   
        for item in list:
            queue.put(item)

 #wait on the queue until everything has been processed     
        queue.join()

main()
print "Elapsed Time: %s" % (time.time() - start)

Sample output:

['host1', 'host2', 'host3']
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.63A

Elapsed Time: 0.634697914124


# python.py | grep Host | sort | uniq -c
     10 Host:host1      LA:00.00A LB:00.00A LC:00.38A
     10 Host:host2      LA:00.00A LB:00.00A LC:00.00A
      5 Host:host3      LA:00.75A LB:01.13A LC:09.50A
      5 Host:host3      LA:00.75A LB:01.13A LC:09.63A

Answer 1

Double-check your indentation.

You enqueue the entirety of your list of hosts every time you initialize a new worker thread. Further, you serialize those threads by join()ing the queue:

for i in xrange(10):
  t = ThreadURL(q)
  ...
  for item in host_list:   # oops
    q.put(item)
  q.join()                 # oops