#自己获取的ip IPPOOLS1 = urllib . request . urlopen ( http : // 127 . 0 . 0 . 1 : 8000 / ?types = 0&count = 20&country = %E5%9B%BD%E5%86%85 ) . read () . decode ( utf - 8 ignore ) IPPOOLS2 = re . findall ( \ ( \ d + \ . \ d + \ . \ d + \ . \ d + \ \ \ s * \ d + ) IPPOOLS1 ) IPPOOL = [ i . replace ( : ) for i in IPPOOLS2 ]

def GetIPPOOLS(num):
    #大象代理买的ip,5元20000个，每十个差不多有一个能用
    IPPOOL=urllib.request.urlopen("http://tpv.daxiangdaili.com/ip/?tid=559480480576119&num="+str(num)+"&operator=1&filter=on&protocol=http&category=2&delay=1").read().decode("utf-8","ignore").split('\r\n')
    '''
    #自己获取的ip
    IPPOOLS1=urllib.request.urlopen("http://127.0.0.1:8000/?types=0&count=20&country=%E5%9B%BD%E5%86%85").read().decode("utf-8",'ignore')
    IPPOOLS2=re.findall('\"(\d+\.\d+\.\d+\.\d+\"\,\s*\d+)',IPPOOLS1)
    IPPOOL=[i.replace('", ',':') for i in IPPOOLS2]
    '''
    return IPPOOL

#大象代理买的ip,5元20000个，每十个差不多有一个能用


把有效的IP存入	REDIS数据库

把有效的IP存入	REDIS数据库

把有效的IP存入 REDIS数据库

def initIPPOOLS(rconn):
    """把有效的IP存入	REDIS数据库"""

    ipNum=len(rconn.keys('IP*'))
    if ipNum<IPPOOLNUM:
        IPPOOLS=GetIPPOOLS(IPPOOLNUM)
        for ipall in IPPOOLS:
            try:
                ip=ipall.split(':')[0]
                port=ipall.split(':')[1]
                telnetlib.Telnet(ip,port=port,timeout=2) #检验代理ip是否有效
            except:
                logger.warning("The ip is not available !( IP:%s )" % ipall)
            else:
                logger.warning("Get ip Success!( IP:%s )" % ipall)
                rconn.set("IP:%s:10"%(ipall),ipall)     #10 is status
    else:
        logger.warning("The number of  the IP is %s!" % str(ipNum))

def
initIPPOOLS
(
rconn
)
:
ipNum
=
len
(
rconn
.
keys
(
'IP*'
)
)
if
ipNum
<
IPPOOLNUM
:
IPPOOLS
=
GetIPPOOLS
(
IPPOOLNUM
)
for
ipall
in
IPPOOLS
:
try
:
ip
=
ipall
.
split
(
':'
)
[
0
]
port
=
ipall
.
split
(
':'
)
[
1
]
telnetlib
.
Telnet
(
ip
,
port
=
port
,
timeout
=
2
)
#检验代理ip是否有效
except
:
logger
.
warning
(
"The ip is not available !( IP:%s )"
%
ipall
)
else
:
logger
.
warning
(
"Get ip Success!( IP:%s )"
%
ipall
)
rconn
.
set
(
"IP:%s:10"
%
(
ipall
)
,
ipall
)
#10 is status
else
:
logger
.
warning
(
"The number of  the IP is %s!"
%
str
(
ipNum
)
)


def updateIPPOOLS(rconn,ip,status,flag=0): # 0代表对status减一，-1代表减2，1代表加1
    if int(status) < 1:
        removeIPPOOLS(rconn,ip,status)
        return
    '''update status'''
    if flag == 1: #+status
        if int(status) < 10:
            rconn.delete('IP:'+ ip + ':' + status)
            status = int(status) + 1
            rconn.set("IP:%s:%s"%(ip,str(status)),ip)
    elif flag == -1:
        rconn.delete('IP:'+ ip + ':' + status)
        status = int(status) - 2
        rconn.set("IP:%s:%s"%(ip,str(status)),ip)
    else:
        rconn.delete('IP:'+ ip + ':' + status)
        status = int(status) - 1
        rconn.set("IP:%s:%s"%(ip,str(status)),ip)

def
updateIPPOOLS
(
rconn
,
ip
,
status
,
flag
=
0
)
:
# 0代表对status减一，-1代表减2，1代表加1
if
int
(
status
)
<
1
:
removeIPPOOLS
(
rconn
,
ip
,
status
)
return
if
flag
==
1
:
#+status
if
int
(
status
)
<
10
:
rconn
.
delete
(
'IP:'
+
ip
+
':'
+
status
)
status
=
int
(
status
)
+
1
rconn
.
set
(
"IP:%s:%s"
%
(
ip
,
str
(
status
)
)
,
ip
)
elif
flag
==
1
:
rconn
.
delete
(
'IP:'
+
ip
+
':'
+
status
)
status
=
int
(
status
)
2
rconn
.
set
(
"IP:%s:%s"
%
(
ip
,
str
(
status
)
)
,
ip
)
else
:
rconn
.
delete
(
'IP:'
+
ip
+
':'
+
status
)
status
=
int
(
status
)
1
rconn
.
set
(
"IP:%s:%s"
%
(
ip
,
str
(
status
)
)
,
ip
)


def initCookie(rconn, spiderName):
    """ 获取所有账号的Cookies，存入Redis。如果Redis已有该账号的Cookie，则不再获取。 """
    for zhihu in myZhiHu:
        if rconn.get("%s:Cookies:%s--%s" % (spiderName, zhihu[0], zhihu[1])) is None:  # 'zhihuspider:Cookies:账号--密码'，为None即不存在。
            cookie = getCookie(zhihu[0], zhihu[1],zhihu[2])
            if len(cookie) > 0:
                rconn.set("%s:Cookies:%s--%s" % (spiderName, zhihu[0], zhihu[1]), cookie)
    cookieNum = str(rconn.keys()).count("zhihuspider:Cookies")
    logger.warning("The num of the cookies is %s" % cookieNum)
    if cookieNum == 0:
        logger.warning('Stopping...')
        os.system("pause")


更新一个账号的Cookie

更新一个账号的Cookie

更新一个账号的Cookie

def updateCookie(accountText, rconn, spiderName, cookie):
    """ 更新一个账号的Cookie """
    account = accountText.split("--")[0]
    #pdb.set_trace()
    new_cookie = UpdateCookie(account, cookie)
    if len(new_cookie) > 0:
        logger.warning("The cookie of %s has been updated successfully!" % account)
        rconn.set("%s:Cookies:%s" % (spiderName, accountText), new_cookie)
    else:
        logger.warning("The cookie of %s updated failed! Remove it!" % accountText)
        removeCookie(accountText, rconn, spiderName)


删除某个账号的Cookie

删除某个账号的Cookie

删除某个账号的Cookie

def removeCookie(accountText, rconn, spiderName):
    """ 删除某个账号的Cookie """
    rconn.delete("%s:Cookies:%s" % (spiderName, accountText))
    cookieNum = str(rconn.keys()).count("zhihuspider:Cookies")
    logger.warning("The num of the cookies left is %s" % cookieNum)
    if cookieNum == 0:
        logger.warning("Stopping...")
        os.system("pause")

def
removeCookie
(
accountText
,
rconn
,
spiderName
)
:
rconn
.
delete
(
"%s:Cookies:%s"
%
(
spiderName
,
accountText
)
)
cookieNum
=
str
(
rconn
.
keys
(
)
)
.
count
(
"zhihuspider:Cookies"
)
logger
.
warning
(
"The num of the cookies left is %s"
%
cookieNum
)
if
cookieNum
==
0
:
logger
.
warning
(
"Stopping..."
)
os
.
system
(
"pause"
)


def __init__(self, server,
                 persist=False,
                 flush_on_start=False,
                 queue_key=defaults.SCHEDULER_QUEUE_KEY,
                 queue_cls=defaults.SCHEDULER_QUEUE_CLASS,
                 dupefilter_key=defaults.SCHEDULER_DUPEFILTER_KEY,
                 dupefilter_cls=defaults.SCHEDULER_DUPEFILTER_CLASS,
                 idle_before_close=0,
                 serializer=None):
        """Initialize scheduler.

        Parameters
        ----------
        server : Redis
            The redis server instance.
        persist : bool
            Whether to flush requests when closing. Default is False.
        flush_on_start : bool
            Whether to flush requests on start. Default is False.
        queue_key : str
            Requests queue key.
        queue_cls : str
            Importable path to the queue class.
        dupefilter_key : str
            Duplicates filter key.
        dupefilter_cls : str
            Importable path to the dupefilter class.
        idle_before_close : int
            Timeout before giving up.

        """
        if idle_before_close < 0:
            raise TypeError("idle_before_close cannot be negative")

        self.server = server
        self.persist = persist
        self.flush_on_start = flush_on_start
        self.queue_key = queue_key
        self.queue_cls = queue_cls
        self.dupefilter_cls = dupefilter_cls
        self.dupefilter_key = dupefilter_key
        self.idle_before_close = idle_before_close
        self.serializer = serializer
        self.stats = None

def
__init__
(
self
,
server
,
persist
=
False
,
flush_on_start
=
False
,
queue_key
=
defaults
.
SCHEDULER_QUEUE_KEY
,
queue_cls
=
defaults
.
SCHEDULER_QUEUE_CLASS
,
dupefilter_key
=
defaults
.
SCHEDULER_DUPEFILTER_KEY
,
dupefilter_cls
=
defaults
.
SCHEDULER_DUPEFILTER_CLASS
,
idle_before_close
=
0
,
serializer
=
)
:
if
idle_before_close
<
0
:
raise
TypeError
(
"idle_before_close cannot be negative"
)
self
.
server
=
server
self
.
persist
=
persist
self
.
flush_on_start
=
flush_on_start
self
.
queue_key
=
queue_key
self
.
queue_cls
=
queue_cls
self
.
dupefilter_cls
=
dupefilter_cls
self
.
dupefilter_key
=
dupefilter_key
self
.
idle_before_close
=
idle_before_close
self
.
serializer
=
serializer
self
.
stats
=


def __init__(self, server,
                 key=defaults.PIPELINE_KEY,
                 serialize_func=default_serialize):
        """Initialize pipeline.

        Parameters
        ----------
        server : StrictRedis
            Redis client instance.
        key : str
            Redis key where to store items.
        serialize_func : callable
            Items serializer function.

        """
        self.server = server
        self.key = key
        self.serialize = serialize_func

def __init__ ( self , server , key = defaults . PIPELINE_KEY , serialize_func = default_serialize ) : self . server = server self . key = key self . serialize = serialize_func


Returns redis key based on given spider .

def item_key(self, item, spider):
        """Returns redis key based on given spider.

        Override this function to use a different key depending on the item
        and/or spider.

        """
        return self.key % {'spider': spider.name}

def item_key ( self , item , spider ) : return self . key % { 'spider' : spider . name }


def __init__(self, server, spider, key, serializer=None):
        """Initialize per-spider redis queue.

        Parameters
        ----------
        server : StrictRedis
            Redis client instance.
        spider : Spider
            Scrapy spider instance.
        key: str
            Redis key where to put and get messages.
        serializer : object
            Serializer object with ``loads`` and ``dumps`` methods.

        """
        if serializer is None:
            # Backward compatibility.
            # TODO: deprecate pickle.
            serializer = picklecompat
        if not hasattr(serializer, 'loads'):
            raise TypeError("serializer does not implement 'loads' function: %r"
                            % serializer)
        if not hasattr(serializer, 'dumps'):
            raise TypeError("serializer '%s' does not implement 'dumps' function: %r"
                            % serializer)

        self.server = server
        self.spider = spider
        self.key = key % {'spider': spider.name}
        self.serializer = serializer


def _encode_request(self, request):
        """Encode a request object"""
        obj = request_to_dict(request, self.spider)
        return self.serializer.dumps(obj)

def _encode_request ( self , request ) : obj = request_to_dict ( request , self . spider ) return self . serializer . dumps ( obj )


def _decode_request(self, encoded_request):
        """Decode an request previously encoded"""
        obj = self.serializer.loads(encoded_request)
        return request_from_dict(obj, self.spider)

def _decode_request ( self , encoded_request ) : obj = self . serializer . loads ( encoded_request ) return request_from_dict ( obj , self . spider )


def __len__(self):
        """Return the length of the queue"""
        raise NotImplementedError

def __len__ ( self ) : raise NotImplementedError


def push(self, request):
        """Push a request"""
        raise NotImplementedError

def push ( self , request ) : raise NotImplementedError


def pop(self, timeout=0):
        """Pop a request"""
        raise NotImplementedError

def pop ( self , timeout = 0 ) : raise NotImplementedError


def clear(self):
        """Clear queue/stack"""
        self.server.delete(self.key)

def clear ( self ) : self . server . delete ( self . key )


def __len__(self):
        """Return the length of the queue"""
        return self.server.llen(self.key)

def __len__ ( self ) : return self . server . llen ( self . key )


def push(self, request):
        """Push a request"""
        self.server.lpush(self.key, self._encode_request(request))

def push ( self , request ) : self . server . lpush ( self . key , self . _encode_request ( request ) )


def pop(self, timeout=0):
        """Pop a request"""
        if timeout > 0:
            data = self.server.brpop(self.key, timeout)
            if isinstance(data, tuple):
                data = data[1]
        else:
            data = self.server.rpop(self.key)
        if data:
            return self._decode_request(data)

def pop ( self , timeout = 0 ) : if timeout > 0 : data = self . server . brpop ( self . key , timeout ) if isinstance ( data , tuple ) : data = data [ 1 ] else : data = self . server . rpop ( self . key ) if data : return self . _decode_request ( data )


def __len__(self):
        """Return the length of the queue"""
        return self.server.zcard(self.key)

def __len__ ( self ) : return self . server . zcard ( self . key )


def push(self, request):
        """Push a request"""
        data = self._encode_request(request)
        score = -request.priority
        # We don't use zadd method as the order of arguments change depending on
        # whether the class is Redis or StrictRedis, and the option of using
        # kwargs only accepts strings, not bytes.
        self.server.execute_command('ZADD', self.key, score, data)


def pop(self, timeout=0):
        """
        Pop a request
        timeout not support in this queue class
        """
        # use atomic range/remove using multi/exec
        pipe = self.server.pipeline()
        pipe.multi()
        pipe.zrange(self.key, 0, 0).zremrangebyrank(self.key, 0, 0)
        results, count = pipe.execute()
        if results:
            return self._decode_request(results[0])


def __len__(self):
        """Return the length of the stack"""
        return self.server.llen(self.key)


def pop(self, timeout=0):
        """Pop a request"""
        if timeout > 0:
            data = self.server.blpop(self.key, timeout)
            if isinstance(data, tuple):
                data = data[1]
        else:
            data = self.server.lpop(self.key)

        if data:
            return self._decode_request(data)

def pop ( self , timeout = 0 ) : if timeout > 0 : data = self . server . blpop ( self . key , timeout ) if isinstance ( data , tuple ) : data = data [ 1 ] else : data = self . server . lpop ( self . key ) if data : return self . _decode_request ( data )


Returns a redis client instance from given Scrapy settings object .

def get_redis_from_settings(settings):
    """Returns a redis client instance from given Scrapy settings object.

    This function uses ``get_client`` to instantiate the client and uses
    ``defaults.REDIS_PARAMS`` global as defaults values for the parameters. You
    can override them using the ``REDIS_PARAMS`` setting.

    Parameters
    ----------
    settings : Settings
        A scrapy settings object. See the supported settings below.

    Returns
    -------
    server
        Redis client instance.

    Other Parameters
    ----------------
    REDIS_URL : str, optional
        Server connection URL.
    REDIS_HOST : str, optional
        Server host.
    REDIS_PORT : str, optional
        Server port.
    REDIS_ENCODING : str, optional
        Data encoding.
    REDIS_PARAMS : dict, optional
        Additional client parameters.

    """
    params = defaults.REDIS_PARAMS.copy()
    params.update(settings.getdict('REDIS_PARAMS'))
    # XXX: Deprecate REDIS_* settings.
    for source, dest in SETTINGS_PARAMS_MAP.items():
        val = settings.get(source)
        if val:
            params[dest] = val

    # Allow ``redis_cls`` to be a path to a class.
    if isinstance(params.get('redis_cls'), six.string_types):
        params['redis_cls'] = load_object(params['redis_cls'])

    return get_redis(**params)


def get_redis(**kwargs):
    """Returns a redis client instance.

    Parameters
    ----------
    redis_cls : class, optional
        Defaults to ``redis.StrictRedis``.
    url : str, optional
        If given, ``redis_cls.from_url`` is used to instantiate the class.
    **kwargs
        Extra parameters to be passed to the ``redis_cls`` class.

    Returns
    -------
    server
        Redis client instance.

    """
    redis_cls = kwargs.pop('redis_cls', defaults.REDIS_CLS)
    url = kwargs.pop('url', None)
    if url:
        return redis_cls.from_url(url, **kwargs)
    else:
        return redis_cls(**kwargs)

def get_redis ( * * kwargs ) : redis_cls = kwargs . pop ( 'redis_cls' , defaults . REDIS_CLS ) url = kwargs . pop ( 'url' , None ) if url : return redis_cls . from_url ( url , * * kwargs ) else : return redis_cls ( * * kwargs )


Returns a batch of start requests from redis .

def start_requests(self):
        """Returns a batch of start requests from redis."""
        return self.next_requests()

def start_requests ( self ) : return self . next_requests ( )


Setup redis connection and idle signal .

def setup_redis(self, crawler=None):
        """Setup redis connection and idle signal.

        This should be called after the spider has set its crawler object.
        """
        if self.server is not None:
            return

        if crawler is None:
            # We allow optional crawler argument to keep backwards
            # compatibility.
            # XXX: Raise a deprecation warning.
            crawler = getattr(self, 'crawler', None)

        if crawler is None:
            raise ValueError("crawler is required")

        settings = crawler.settings

        if self.redis_key is None:
            self.redis_key = settings.get(
                'REDIS_START_URLS_KEY', defaults.START_URLS_KEY,
            )

        self.redis_key = self.redis_key % {'name': self.name}

        if not self.redis_key.strip():
            raise ValueError("redis_key must not be empty")

        if self.redis_batch_size is None:
            # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).
            self.redis_batch_size = settings.getint(
                'REDIS_START_URLS_BATCH_SIZE',
                settings.getint('CONCURRENT_REQUESTS'),
            )

        try:
            self.redis_batch_size = int(self.redis_batch_size)
        except (TypeError, ValueError):
            raise ValueError("redis_batch_size must be an integer")

        if self.redis_encoding is None:
            self.redis_encoding = settings.get('REDIS_ENCODING', defaults.REDIS_ENCODING)

        self.logger.info("Reading start URLs from redis key '%(redis_key)s' "
                         "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",
                         self.__dict__)

        self.server = connection.from_settings(crawler.settings)
        # The idle signal is called when the spider has no requests left,
        # that's when we will schedule new requests from redis queue
        crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)


Returns a request to be scheduled or none .

def next_requests(self):
        """Returns a request to be scheduled or none."""
        use_set = self.settings.getbool('REDIS_START_URLS_AS_SET', defaults.START_URLS_AS_SET)
        fetch_one = self.server.spop if use_set else self.server.lpop
        # XXX: Do we need to use a timeout here?
        found = 0
        # TODO: Use redis pipeline execution.
        while found < self.redis_batch_size:
            data = fetch_one(self.redis_key)
            if not data:
                # Queue empty.
                break
            req = self.make_request_from_data(data)
            if req:
                yield req
                found += 1
            else:
                self.logger.debug("Request not made from data: %r", data)

        if found:
            self.logger.debug("Read %s requests from '%s'", found, self.redis_key)


Returns a Request instance from data coming from Redis .

def make_request_from_data(self, data):
        """Returns a Request instance from data coming from Redis.

        By default, ``data`` is an encoded URL. You can override this method to
        provide your own message decoding.

        Parameters
        ----------
        data : bytes
            Message from redis.

        """
        url = bytes_to_str(data, self.redis_encoding)
        return self.make_requests_from_url(url)

def make_request_from_data ( self , data ) : url = bytes_to_str ( data , self . redis_encoding ) return self . make_requests_from_url ( url )


def schedule_next_requests(self):
        """Schedules a request if available"""
        # TODO: While there is capacity, schedule a batch of redis requests.
        for req in self.next_requests():
            self.crawler.engine.crawl(req, spider=self)


Schedules a request if available otherwise waits .

def spider_idle(self):
        """Schedules a request if available, otherwise waits."""
        # XXX: Handle a sentinel to close the spider.
        self.schedule_next_requests()
        raise DontCloseSpider


Returns a str if a bytes object is given .

def bytes_to_str(s, encoding='utf-8'):
    """Returns a str if a bytes object is given."""
    if six.PY3 and isinstance(s, bytes):
        return s.decode(encoding)
    return s

def bytes_to_str ( s , encoding = 'utf-8' ) : if six . PY3 and isinstance ( s , bytes ) : return s . decode ( encoding ) return s


def __init__(self, server, key, debug=False):
        """Initialize the duplicates filter.

        Parameters
        ----------
        server : redis.StrictRedis
            The redis server instance.
        key : str
            Redis key Where to store fingerprints.
        debug : bool, optional
            Whether to log filtered requests.

        """
        self.server = server
        self.key = key
        self.debug = debug
        self.bf = BloomFilter(server, key, blockNum=1)  # you can increase blockNum if your are filtering too many urls
        self.logdupes = True


Returns an instance from given settings .

def from_settings(cls, settings):
        """Returns an instance from given settings.

        This uses by default the key ``dupefilter:<timestamp>``. When using the
        ``scrapy_redis.scheduler.Scheduler`` class, this method is not used as
        it needs to pass the spider name in the key.

        Parameters
        ----------
        settings : scrapy.settings.Settings

        Returns
        -------
        RFPDupeFilter
            A RFPDupeFilter instance.

        """
        server = get_redis_from_settings(settings)
        # XXX: This creates one-time key. needed to support to use this
        # class as standalone dupefilter with scrapy's default scheduler
        # if scrapy passes spider on open() method this wouldn't be needed
        # TODO: Use SCRAPY_JOB env as default and fallback to timestamp.
        key = defaults.DUPEFILTER_KEY % {'timestamp': int(time.time())}
        debug = settings.getbool('DUPEFILTER_DEBUG')
        return cls(server, key=key, debug=debug)


def from_crawler(cls, crawler):
        """Returns instance from crawler.

        Parameters
        ----------
        crawler : scrapy.crawler.Crawler

        Returns
        -------
        RFPDupeFilter
            Instance of RFPDupeFilter.

        """
        return cls.from_settings(crawler.settings)

def from_crawler ( cls , crawler ) : return cls . from_settings ( crawler . settings )


Returns True if request was already seen .

def request_seen(self, request):
        """Returns True if request was already seen.

        Parameters
        ----------
        request : scrapy.http.Request

        Returns
        -------
        bool

        """
        fp = request_fingerprint(request)
        if self.bf.isContains(fp):
            return True
        else:
            self.bf.insert(fp)
            return False

def request_seen ( self , request ) : fp = request_fingerprint ( request ) if self . bf . isContains ( fp ) : return True else : self . bf . insert ( fp ) return False


Returns a fingerprint for a given request .

def request_fingerprint(self, request):
        """Returns a fingerprint for a given request.

        Parameters
        ----------
        request : scrapy.http.Request

        Returns
        -------
        str

        """
        return request_fingerprint(request)

def request_fingerprint ( self , request ) : return request_fingerprint ( request )


Delete data on close . Called by Scrapy s scheduler .

def close(self, reason=''):
        """Delete data on close. Called by Scrapy's scheduler.

        Parameters
        ----------
        reason : str, optional

        """
        self.clear()

def close ( self , reason = '' ) : self . clear ( )


def clear(self):
        """Clears fingerprints data."""
        self.server.delete(self.key)

def clear ( self ) : self . server . delete ( self . key )


def log(self, request, spider):
        """Logs given request.

        Parameters
        ----------
        request : scrapy.http.Request
        spider : scrapy.spiders.Spider

        """
        if self.debug:
            msg = "Filtered duplicate request: %(request)s"
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
        elif self.logdupes:
            msg = ("Filtered duplicate request %(request)s"
                   " - no more duplicates will be shown"
                   " (see DUPEFILTER_DEBUG to show all duplicates)")
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
            self.logdupes = False