#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import with_statement

"""niconicodl.py: niconico downloader """
__version__ = "1.3"
__revision__ = "$Rev: 1 $"
__license__ = "public domain"
__author__ = "T.A"
__contributors__ = "?"

################################################################################

NICO_BASE_URL  = "http://www.nicovideo.jp/"

# NICO_LOGIN_URL = NICO_BASE_URL # +"login"
NICO_LOGIN_URL = 'https://secure.nicovideo.jp/secure/login?site=niconico'
# NICO_LOGOUT_URL = NICO_BASE_URL+"logout"
NICO_LOGOUT_URL = 'https://secure.nicovideo.jp/secure/logout'

NEWURL_LIST = [NICO_BASE_URL + "newarrival",
               NICO_BASE_URL + "newarrival?from=31",
               NICO_BASE_URL + "newarrival?from=61",
               NICO_BASE_URL + "newarrival?from=91",
               NICO_BASE_URL + "newarrival?from=121",
               NICO_BASE_URL + "newarrival?from=151",
               NICO_BASE_URL + "newarrival?from=181",
               NICO_BASE_URL + "newarrival?from=211",
               NICO_BASE_URL + "newarrival?from=241",
               NICO_BASE_URL + "newarrival?from=271" ]

NICO_NEW_URL   = NICO_BASE_URL + "newarrival"

NICO_RANK_URL  = NICO_BASE_URL + "ranking/mylist/daily/all"

NICO_WATCH_URL = NICO_BASE_URL+"watch/"

NICO_TAG_URL   = NICO_BASE_URL+"tag/"

NICO_FIND_URL  = NICO_BASE_URL+"search/"

NICO_API_URL   = NICO_BASE_URL+"api/getflv?v=%s"

ALL_RETRIEVE     = ['xml','html','flv']
SORT_OPTION_DIC  = { 'comment_amount' : 'r',
                     'comment_date'   : 'n',
                     'regist_date'    : 'f',
                     'view_count'     : 'v'
                     }

USER_AGENT = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)'

import sys
import urllib
import urllib2
import cookielib
import re
import cgi
import time
import os
import signal
import string
import traceback
import logging

import threading

import httplib # BaseStatusLine
# import doctest
from progressbar import *
# external setting file read.
import niconicodl_cfg as niconfig

# Constants for progress bar
PBAR_WIDGETS = [Percentage(), " ", Bar(marker="=",left="[",right="]"),
           " ", ETA(), " ", FileTransferSpeed()]

##
def nico_exception_deco(f):
    def skelton(self):
        f(self)
    return skelton

##
def nico_init_exception_deco(f):
    def skelton(self, var):
        self.message = "[" + var + "]"
        f(self, var)
    return skelton

class NicoException(Exception):
    @nico_init_exception_deco
    def __init__(self, msg): print("[normal exception]")

class CriticalException(Exception):
    @nico_init_exception_deco
    def __init__(self, msg): print("[critical exception]")

class WarningException(Exception):
    @nico_init_exception_deco
    def __init__(self, msg): print("[warning exception]")

class ALockedException(Exception):
    @nico_init_exception_deco
    def __init__(self, msg): print("[access locked exception]")

class PushableIterator:
    def __init__(self, ii):
        self._ii = iter(ii)
        self._pushBuf = []
        return

    def __iter__(self):
        return self

    def next(self):
        if self._pushBuf:
            return self._pushBuf.pop(0)
        return self._ii.next()

    def push(self, x):
        self._pushBuf.insert(0, x)

class EncStr(object):
    """ string with encode decode """
    def __init__(self, arg):
        self.__str   = arg
        self.__fsenc = niconfig.FS_ENCODING

    def __add__(self, v): return self.__str + v

    def sets(self,v):
        self.__str = v
        return self.__str

    def gets(self): return self.__str

    def dels(self): del self.__str

    def set_from_utf8(self,v):
        try:
            self.__str = v.decode('utf-8', 'ignore')
        except:
            logging.exception('decode to utf-8')
        return self.__str

    def set_from_fsenc(self,v):
        try:
            self.__str = v.decode(self.__fsenc, 'ignore')
        except:
            logging.exception('decode to %s' % self.__fsenc)
        return self.__str

    def get_utf8(self):
        r = ""
        try:
            r = self.__str.encode('utf-8', 'ignore')
        except:
            logging.exception('encode to utf-8')
        return r

    def get_fsenc(self):
        r = ""
        try:
            r = self.__str.encode(self.__fsenc, 'ignore')
        except:
            logging.exception('encode to %s' % self.__fsenc)
        return r

    __slots__ = ('__str', '__fsenc')
    string = property(gets, sets, dels, "string property.")

#
# main class
#
class NicoDownloader(object):
    """ for niconico douga download """

    #__slots__ = (
    #    'new_mode',
    #    'mask_mode',
    #    'debug_mode',
    #    'daemon_mode',
    #    'test_mode',
    #    'search_words',
    #    'tag_mode',
    #    'retrieve_limit',
    #    'retrieve_item',
    #
    #    '_basename',
    #    '_htm',
    #    '_xml',
    #    '_flv',
    #    '_store',
    #    '_opener',
    #    '_quit',
    #
    #    'log',
    #    )

    # @staticmethod ...
    @classmethod
    def set_quit(cls):
        cls._quit = True

    retrieved_counter = { 'xml' : 0, 'flv' : 0, 'html' : 0 }

    #
    def __init__(self):
        self._opener      = None
        self._req         = None
        self._quit        = False

        self._basename    = EncStr("")
        self._htm         = EncStr("")
        self._xml         = EncStr("")
        self._flv         = EncStr("")
        self._store       = EncStr("")

        self.close_req = False # maybe must be False...

    #
    def sleep(self, second):
        time.sleep(second)
        return

    #
    def time_str(self):
        return time.strftime("%Y-%m-%d_%H%M%S", time.localtime())

    #
    def get_nid_from_url(self, url):
        pat = re.compile(NICO_WATCH_URL+'(.*)')
        m = pat.match(url)
        if m:
            return m.group(1)
        else:
            return None

    #
    def set_filename(self):
        self._flv.string = self._basename + '.flv'
        self._htm.string = self._basename + '.html'
        self._xml.string = self._basename + '.xml'
        self._store.string = self._basename + '.txt'

    #
    def make_all_filename(self, ntitle, nid, flv_url):
        try:
            logging.info("make all filename: (" + ntitle.encode('utf-8', 'ignore') + ")")
        except:
            logging.exception("")

        dest = niconfig.SAVE_DESTINATION
        if self.save_dest and len(self.save_dest) > 0:
            dest = self.save_dest
        if dest[-1] != '/':
            dest = dest + '/'
        if not os.path.isdir(dest):
            try:
                os.makedirs(dest)
            except:
                logging.exception("makedirs: " + dest)
                raise CriticalException("makedirs")

        base = dest + ntitle

        if flv_url[-3:] == 'low':
            logging.info("low mode")
            if self.notdllow_mode:
                logging.info("skip: low mode movie[%s]" % flv_url)
                return False
            base = base + "_low"

        self._basename.string = base

        self.set_filename()

        if self.force_save: # force save mode
            return True

        while os.path.exists(self._flv.get_fsenc()) or os.path.exists(self._htm.get_fsenc()) or os.path.exists(self._xml.get_fsenc()):
            base = base + "+"
            self._basename.string = base
            self.set_filename()

        return True

    #
    def output_html(self, content):
        if not 'html' in self.retrieve_item:
            return
        if self.test_mode:
            self.retrieved_countup('html')
            return
        try:
            f = file(self._htm.get_fsenc(), 'w')
            print >>f, content.encode('utf-8', 'ignore')
            f.close()
            self.retrieved_countup('html')
        except:
            self.store_contents(content)
            logging.exception("file, encode:" + + self._htm.get_utf8())

    #
    def store_contents(self, filename, addstr, con):
        try:
            f = file(filename.get_fsenc() + addstr, 'w')
            f.write(con)
            f.close()
            logging.info("stored contents to %s" % filename.get_utf8())
        except:
            logging.exception("store file")

    #
    def get_title(self, url, nid, flv_url):
        logging.info("get_title("+url+")")
        con = None
        try:
            con = self.get(url)
            
        except:
            logging.exception("get")
            return None
        try:
            content = con.decode('utf-8', 'ignore')
        except:
            self.store_contents(self._store, "", con)
            logging.exception("decode")
            return None

        pat = re.compile(r'<title.*?>(.*?)</title>', re.M)
        m = pat.search(content)
        
        if m:
            title = m.group(1)
            title = self.normalize(title)
            title = nid + " - " + title.strip(u"‐ ニコニコ動画(原宿)")
            print title.encode("gb18030")

            res = self.make_all_filename(title, nid, flv_url)
            if res == False: # not dl low mode movie
                return None

            if not self.mask_mode:
                self.output_html(content)
                return title
            else:
                for cmpstr in niconfig.MASK_LIST:
                    rtitle = title # .encode('utf-8', 'ignore')
                    if rtitle.find(cmpstr) != -1:
                        self.output_html(content)
                        return title
                try:
                    logging.info("skip: " + title.encode('utf-8', 'ignore'))
                except:
                    logging.exception("encode")
                return None
        else:
            logging.info("!!! not found title tag in: (" + url + ")")
            return None

    #
    def get_flv_url(self, nid):
        logging.info("get_flv_url("+nid+")")
        flv_url = NICO_API_URL % urllib2.quote(nid)
        url = None
        try:
            r = self.get(flv_url)
            logging.debug(r)
            url = cgi.parse_qs(r)
            # print url
            return url['url'][0], url['thread_id'][0], url['ms'][0]
        except:
            logging.error(url)
            logging.exception("get, cgi.parse, url['...']")

            if url and url.has_key('error') and 'flv_get_error' in url['error']:
                raise WarningException("flv_get_error")
            if url and url.has_key('error') and 'access_locked' in url['error']:
                raise ALockedException("access_locked")
            if url and url.has_key('error') and 'cant_get_detail' in url['error']:
                raise WarningException("cant_get_detail")
            logging.error("get timeout ?")
            raise NicoException("timeout?")

    #
    def get(self, url):
        logging.debug("get:" + url)
        try:
            fp = self._opener.open(url)
            logging.debug(fp.info())
        except urllib2.URLError, e:
            logging.exception("opener.open raise URLError [%s]" % e.message)
            raise NicoException("get:opener.open")
        except:
            logging.exception("opener.open: ")
            raise NicoException("get:opener.open")
        try:
            content = fp.read()
            return content
        finally:
            fp.close()

    #
    def check_already_dl(self, nid, totalsize):
        """ if already donwload then return True """
        if not os.path.exists(niconfig.FLV_HISTORY):
            return False
        f = file(niconfig.FLV_HISTORY, 'r')
        alreadyDL = False
        for idx, urlsiz in enumerate([x.strip() for x in f]):
            if urlsiz == "%s:%d" % (nid, totalsize): # perfect match
                alreadyDL = True
                break
            else:
                smid = urlsiz[:urlsiz.find(':')]
                hsiz = int(urlsiz[urlsiz.find(':')+1:])
                # ID match and size is low then
                if smid == nid and hsiz != totalsize:
                    logging.info("download (%d != %d): " % (hsiz, totalsize))
                    alreadyDL = False
        f.close()

        return alreadyDL

    #
    def save_file(self, reader, writer):
        b = reader.read()
        writer.write(b)
        return

    #
    def remove_html(self):
        if os.path.exists(self._htm.get_fsenc()):
            os.unlink(self._htm.get_fsenc())
        return

    #
    def retrieved_countup(self, item):
        self.retrieved_counter[item] = self.retrieved_counter[item] + 1
        if self.retrieve_limit != 0 and self.retrieve_limit <= self.retrieved_counter[item]:
            logging.info("limit over then retrieve end (%d/%d)[%s]." % (self.retrieve_limit, self.retrieved_counter[item], item))
            self._quit = True
        return

    #
    def retrieve_flv(self, url, nid, title):
        if not 'flv' in self.retrieve_item:
            return True
        logging.info("retrieve_flv [%s]" % nid)
        try:
            reader = self._opener.open(url)
            logging.debug(reader.info())
        except httplib.BadStatusLine, e:
            logging.exception("opener.open raise BadStatusLine [%s]" % e.message)
            raise WarningException("retrieve_flv: opener.open")
        except:
            logging.exception("opener.open: ")
            raise NicoException("retrieve_flv: opener.open")

        global total #全局变量文件大小
        total = int(reader.headers.get('content-length'))
        print "FileSize: " + str(round(float(total)/1024/1024,2)) + "MB"

        logging.info("=" * 80)
        logging.info("Time    : " + self.time_str())
        logging.info("Video id: " + nid)
        logging.info("Url     : " + url)
        logging.info("Filename: " + self._flv.get_utf8())
        logging.info("Size    : %d" % total)

        if self.check_already_dl(nid, total):
            logging.info("already dl (%s:%d)" % (nid, total))
            logging.info("-" * 80)
            reader.close()
            self.remove_html()
            return False # already dl

        logging.info("-" * 80)

        if self.test_mode:
            reader.close()
            self.retrieved_countup('flv')
            return True

        # self.save_file(reader, writer)
        b = ''
        u = ''
        pbar = ProgressBar(widgets=PBAR_WIDGETS, maxval=int(total)).start()
        while True:
            u = reader.read(niconfig.READ_SIZE)
            if not u:
                break
            b += u
            pbar.update(len(b))
        reader.close()
        pbar.finish()
        for ignoreitem in niconfig.IGNORE_FLV_FILES:
            if os.path.exists(ignoreitem):
                f = file(ignoreitem, 'r')
                igc = f.read()
                f.close()
                if igc == b:
                    logging.info('This is equal to ignore flv [%s]' % ignoreitem)
                    self.remove_html()
                    return False

        writer = file(self._flv.get_fsenc(), 'w')
        writer.write(b)

        writer.close()

        h = file(niconfig.FLV_HISTORY, 'a')
        print >>h, "%s:%d" % (nid, total)
        h.close()

        self.retrieved_countup('flv')

        return True

    #
    def retrieve_xml(self,  ms, th_id):
        if not 'xml' in self.retrieve_item:
            return
        logging.info("retrieve_xml [%s]" % ms)
        try:
            r = urllib2.Request(ms)
            r.add_data(r'<thread res_from="-500" version="20061206" thread="'+th_id+r'"/>')
            logging.debug("get xml url:%s threadid:%s" % (ms, th_id))
            reader = self._opener.open(r)
            logging.debug(reader.info())
        except:
            logging.exception("retrieve_xml: opener.open: ")
            return

        if self.test_mode:
            reader.close()
            self.retrieved_countup('xml')
            return

        writer = file(self._xml.get_fsenc(), 'w')
        self.save_file(reader, writer)
        reader.close()
        writer.close()
        self.retrieved_countup('xml')
        return

    #
    def login(self):

        usern = niconfig.USERNAME
        passw = niconfig.PASSWORD

        if self.chuser and len(self.chuser) > 0:
            if niconfig.USER_AND_PASS_DIC.has_key(self.chuser):
                usern = self.chuser
                passw = niconfig.USER_AND_PASS_DIC[self.chuser]
                logging.debug("change user to [%s]" % usern)
            else:
                logging.info("not found [%s] in USER_AND_PASS_DIC" % self.chuser)

        logging.info("login [%s]" % usern)

        if len(usern) == 0 or len(passw) == 0:
            print("username or password is not set")
            logging.error('username or password is not set')
            self._req = None
            sys.exit(1) # force end.

        postdata = {}
        postdata['mail']     = usern
        postdata['password'] = passw
        postdata['next_url'] = ""
        # postdata['submit.x'] = 
        # postdata['submit.y'] = 

        params = urllib.urlencode(postdata)
        cj = cookielib.CookieJar()
        self._opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        self._opener.addheaders = [('User-agent', USER_AGENT)]

        try:
            def saveloginhtml(req):
                if callable(req.read):
                    f = file('login.html', 'w')
                    f.write(req.read())
                    f.close()
                return

            logging.debug("open: " + NICO_LOGIN_URL)
            self._req = self._opener.open(NICO_LOGIN_URL, params)
            # logging.debug(self._req.geturl())

            if self.debug_mode:
                saveloginhtml(self._req)

            if callable(self._req.info):
                logging.debug(self._req.info())
                if not self._req.info().has_key('x-niconico-authflag') or self._req.info()['x-niconico-authflag'] == '0':
                    logging.exception(self._req.info())
                    saveloginhtml(self._req)
                    raise NicoException("login: x-niconico-authflag not set or 0")

            if self.close_req:
                self._req.close() # TEST
        except urllib2.HTTPError, e:
            logging.exception("login failed [%s]" % e.info())
            self._req = None

        except Exception, e:
            logging.exception("login failed [%s]" % e.message)
            self._req = None

    #
    def logout(self):
        logging.info("logout")
        req = None
        try:
            NICO_LOGOUT_URL
            logging.debug("open: " + NICO_LOGOUT_URL)
            req = self._opener.open(NICO_LOGOUT_URL)
        except:
            logging.exception("logout failed ?")

        if req:
            logging.debug("$" * 80)
            logging.debug(req.info())
            logging.debug("$" * 80)
            req.close()

        if not self.close_req:
          self._req.close() # maybe logout
          logging.debug("$" * 80)
          logging.debug(self._req.info())
          logging.debug("$" * 80)

    #
    def normalize(self, title):
        try:
            title = title.encode('utf-8', 'ignore')

            if False: # why...
                tbl = string.maketrans(u'\\/:*?"<>|;%!&^', u'￥／：＊？” ＜ ＞｜；％！＆＾')
                title.translate(tbl)

            if True:
                # for Windows
                title = title.replace('\\','￥')
                title = title.replace('/','／')
                title = title.replace(':','：')
                title = title.replace('*','＊')
                title = title.replace('?','？')
                title = title.replace('"','”')
                title = title.replace('<', '＜')
                title = title.replace('>', '＞')
                title = title.replace('|','｜')
                # for unix ?
                title = title.replace(';','；')
                title = title.replace('%','％')
                title = title.replace('!','！')
                title = title.replace('&','＆')
                title = title.replace('^','＾')

            title = title.decode('utf-8', 'ignore')
        except:
            logging.exception("encode -> utf-8 or decode -> utf-8")

        return title

    #
    def get_nid_from_html(self, contents):
        logging.debug("get_nid_from_html")
        nids = []
        # pat = re.compile(r'<p><a href="watch/([^>]+)">.*')
        # pat = re.compile(r'.*<p><a href=".*watch/([^>]+)">.*')
        pat = re.compile(r'.*<a[^>]*href=".*watch/([^>]+)">.*')
        for l in contents.split('\n'):
            m = pat.match(l)
            if m:
                nidurl = NICO_WATCH_URL + m.group(1)
                if nidurl in nids:
                    pass
                else:
                    nids.append(nidurl)

        logging.info("got nids: " + ",".join([x[len(NICO_WATCH_URL):] for x in nids]))

        return nids

    #
    def get_next_page(self, cont):
        # <td><img src="http://res.nicovideo.jp/img/common/pager_next_on.gif" alt="次のページ"></a></td>

        pat = re.compile(r'.*<td><a href="([^<>].*)"><img src="http://res.nicovideo.jp/img/common/pager_next_on.gif" alt="次のページ"></a></td>')
        for l in cont.split('\n'):
            m = pat.match(l)
            if m:
                logging.debug('next page : ' + m.group(1))
                return m.group(1)
        logging.info('not found nextpage')
        return None

    #
    def get_sort_types(self, mode):
        stype = ''

        # stype = '/1/n/d/' old type
        # stype = '&k=n&o=d' old type
        if mode == 'tag' or mode == 'find':
            stype = '?page=1&sort=n&order=d'

        # descend -> default
        # asc  -> designate
        if len(stype) and self.sort_types and len(self.sort_types) > 0:
            for k,v in SORT_OPTION_DIC.iteritems():
                if k in self.sort_types:
                    stype = stype.replace('n', v)

            if 'asc' in self.sort_types:
                stype = stype.replace('=d', '=a')

        return stype

    #
    def get_nico_files(self, nicourls):
        logging.info("get_nico_files()")

        def fix_url(url):
            fixurl = url
            if url.find("/") == -1:
                fixurl = NICO_WATCH_URL + url
            if url.find("/thumb/") != -1:
                fixurl = url.replace('/thumb/', '/watch/')
            if url[:3] == 'ttp':
                fixurl = 'h' + url
            return fixurl

        #
        # URL loop
        #
        piter = PushableIterator(nicourls)
        for url in piter:
            try:
                # mini change
                url = fix_url(url)

                nid = self.get_nid_from_url(url)
                if nid == None:
                    logging.warning("error in get video id ("+url.encode('utf-8', 'ignore')+")")
                    continue

                flv_url, thread_id, ms = self.get_flv_url(nid)
                self.sleep(5)

                # get title and dl html
                title = self.get_title(url, nid, flv_url)
                self.sleep(5)

                if title == None or len(title) == 0:
                    continue

                if self.test_mode: continue

                # dl movie
                res = self.retrieve_flv(flv_url, nid, title)
                self.sleep(5)

                if res != False:
                    # dl comment
                    self.retrieve_xml(ms, thread_id)

                if self._quit: break

                self.sleep(10)

            except NicoException:
                if self._quit: break
                fname = url.encode('utf-8', 'ignore')
                logging.exception("catch normal exception and continue.(sleep(10))[%s]" % fname)
                self.sleep(10)
                logging.info("retry: [%s]" % fname)
                piter.push(url)
                self.remove_html()

            except WarningException:
                if self._quit: break
                logging.exception("catch warning exception skip url and continue. (sleep(10)")
                logging.info("skip: "+url.encode('utf-8', 'ignore'))
                self.sleep(10)

            except ALockedException:
                if self._quit: break
                fname = url.encode('utf-8', 'ignore')
                logging.exception("catch alock exception and logout and login and retry this url [%s]" % fname)
                self.logout()
                logging.info("sleep (%d minutes) for access_locked." % self.alock_interval)
                self.wait(self.alock_interval * 60)
                self.login()
                if self._req == None:
                    loggin.error('login failed...')
                    break
                logging.info("retry: [%s]" % fname)
                piter.push(url)

            except CriticalException:
                logging.exception("catch critical exception and exit")
                break

        return True

    #
    def get_any_urls(self, urlmode, initialurl, urls):
        logging.debug("get_%s_urls" % urlmode)
        nids = []
        url = initialurl

        try:
            pagecnt = 0

            while True:
                logging.info("get("+url+")")
                cont = self.get(url)

                if self.debug_mode: # when debug mode
                    html = EncStr(self.time_str())
                    self.store_contents(html, "_"+urlmode+".html", cont)

                nids.extend(self.get_nid_from_html(cont))
                self.sleep(5) # wait

                pagecnt += 1
                if self.page_limit != 0 and pagecnt >= self.page_limit:
                    logging.info("over page limit (%d/%d)" % (pagecnt, self.page_limit))
                    break

                if urlmode == 'arg':
                    if self.next_with_argurl: # -u
                        nextpage = self.get_next_page(cont)
                        if nextpage != None:
                            url = nextpage
                            continue
                    if len(urls) == 0:
                        break
                    else:
                        url = urls.pop(0)
                else:
                    nextpage = self.get_next_page(cont)
                    if nextpage == None:
                        break
                    # url = NICO_BASE_URL + nextpage
                    url = nextpage

        except:
            logging.exception("get %s movie html failed" % urlmode)

        return nids

    #
    def get_urls(self, args):
        logging.info("get_urls")
        urls = args

        if self.ranking_mode:
            self.urllist_list.extend([NICO_RANK_URL])

        if self.urllist_list and len(self.urllist_list) > 0:
            urls = self.urllist_list
            url = urls.pop(0)
            urls.extend(self.get_any_urls('arg', url, urls))

        if self.tag_mode and len(self.tag_mode) > 0:
            mode = EncStr("")
            mode.set_from_fsenc(self.tag_mode)
            url = NICO_TAG_URL  + mode.get_utf8()
            url = url + self.get_sort_types('tag')
            urls.extend(self.get_any_urls('tag', url, []))

        if self.find_words and len(self.find_words) > 0:
            mode = EncStr("")
            mode.set_from_fsenc(self.find_words[0])
            url = NICO_FIND_URL  + urllib2.quote(mode.get_utf8())
            url = url + self.get_sort_types('find')
            urls.extend(self.get_any_urls('find', url, []))

        if self.new_mode:
            url = NICO_NEW_URL
            urls.extend(self.get_any_urls('new', url, []))

        return urls

    #
    def wait(self, sec):
        logging.info(">" * 80)
        logging.info("sleep(%d)" % sec)
        print("sleep(%d):" % sec,)
        sys.stdout.flush()
        self.sleep(sec)
        print("wakeup")
        sys.stdout.flush()
        logging.info("<" * 80)

    #
    def setopt(self, opts):
        self.new_mode = opts.new
        self.mask_mode = opts.mask
        self.daemon_mode = opts.daemon
        self.debug_mode = opts.debug
        self.notdllow_mode = opts.notdllow
        self.test_mode = opts.test
        self.find_words = opts.find
        self.sort_types = opts.sort
        self.tag_mode = opts.tag
        self.retrieve_limit = opts.limit
        self.page_limit = opts.page
        self.ranking_mode = opts.ranking
        self.next_with_argurl = opts.nexturl
        self.force_save = opts.force
        self.save_dest = opts.dest
        self.chuser = opts.chuser
        self.daemon_not_logout_mode = opts.dnlogout
        self.http_server_mode = opts.server

        if opts.urllist == None or len(opts.urllist) == 0:
            self.urllist_list = []
        else:
            self.urllist_list = opts.urllist

        if opts.ret == None or len(opts.ret) == 0:
            self.retrieve_item = ALL_RETRIEVE
        else:
            self.retrieve_item = opts.ret

        def __retval(optv, defv, atleast):
            retv = optv
            if retv <= 0:
                retv = defv
            if retv < atleast:
                retv = atleast
            return retv

        self.daemon_interval = __retval(opts.dinterval, niconfig.DAEMON_INTERVAL, 10)
        self.alock_interval  = __retval(opts.ainterval, niconfig.ALOCKED_INTERVAL, 5)

        # start logging
        logging.basicConfig(level=logging.DEBUG if self.debug_mode else logging.INFO,
                            format='%(asctime)s %(levelname)s %(message)s',
                            filename='niconicodl.log',
                            filemode='a')
        logging.info("*" * 80)
        logging.info("Start: " + self.time_str())
        logging.debug(opts)

        #
        def __chkoptlist(optarglist, deflist, op):
            import sets # because set(NoneType) is error.
            ag = sets.Set(optarglist)
            df = sets.Set(deflist)
            res = ag - df
            if len(res) > 0:
                for x in res:
                    logging.error("unkown opt arg [%s] for %s" % (x, op))
                    print("unkown opt arg [%s] for %s" % (x, op))
                sys.exit(1)

        __chkoptlist(self.retrieve_item, ALL_RETRIEVE, '-R')
        __chkoptlist(self.sort_types, SORT_OPTION_DIC.keys() + ['asc'], '-S')

    #
    def run(self, args):
        logging.info("main")
        logging.debug(self.retrieve_item)

        self.login()
        if self._req == None:
            return False

        #
        # main loop
        #
        while True:
            # get url list html
            nicourls = self.get_urls(args)
            self.sleep(10)

            # and url loop
            self.get_nico_files(nicourls)

            if self.daemon_mode:
                if not self.daemon_not_logout_mode: self.logout()

                self.wait(self.daemon_interval * 60) # wait for daemon mode

                if not self.daemon_not_logout_mode: self.login()
            else:
                break

        self.logout()

        logging.info("End  : " + self.time_str())
        logging.info("*" * 80)
        return True

def signal_handler(arg1, arg2):
    print("catch signal %d" % arg1)
    NicoDownloader.set_quit()
    nico._quit = True
    if nico._req != None:
        nico.logout()
    sys.exit(1)

def makeopts():
    import optparse

    parser = optparse.OptionParser("niconico downloader")
    parser.add_option('-s', '--server',   action='store_true',  dest='server',  default=False, help="http server mode with cherrypy")
    parser.add_option('-d', '--daemon',   action='store_true',  dest='daemon',  default=False, help="daemon mode")
    parser.add_option('-n', '--new',      action='store_true',  dest='new',     default=False, help="get new file mode")
    parser.add_option('-r', '--ranking',  action='store_true',  dest='ranking', default=False, help="get mylist ranking mode")
    parser.add_option('-m', '--mask',     action='store_true',  dest='mask',    default=False, help="mask (in niconicodl_cfg.py) mode")
    parser.add_option('-o', '--notdllow', action='store_true',  dest='notdllow',default=False, help="don't dl lowmode movie")
    parser.add_option('-l', '--dnlogout', action='store_true',  dest='dnlogout',default=False, help="don't logout when daemon sleep.")
    parser.add_option('-u', '--nexturl',  action='store_true',  dest='nexturl', default=False, help="continue to nextpage when set -U")
    parser.add_option('-f', '--force',    action='store_true',  dest='force',   default=False, help="force when already exist same name file")

    parser.add_option('-U', '--urllist',  action='append',      dest='urllist', type='string', help="movie list urls (this is most strong option)")
    parser.add_option('-F', '--find',     action='append',      dest='find',    type='string', help="find strings")
    parser.add_option('-S', '--sort',     action='append',      dest='sort',    type='string', help="sort types [view_count, comment_date, comment_amount, regist_date][asc]")
    parser.add_option('-T', '--tag',      action='store',       dest='tag',     type='string', help="tag mode")
    parser.add_option('-L', '--limit',    action='store',       dest='limit',   type='int', default=0, help="retrieve limit 0 is infinity")
    parser.add_option('-P', '--page',     action='store',       dest='page',    type='int', default=0, help="retrieve page search limit 0 is infinity")
    parser.add_option('-R', '--retrieve', action='append',      dest='ret',     default=None,  help="[xml, html, flv]")
    parser.add_option('-D', '--dest',     action='store',       dest='dest',    type='string', help="save destination.")
    parser.add_option('-I', '--dinterval',action='store',       dest='dinterval',type='int', default=0, help="daemon interval (min).")
    parser.add_option('-A', '--ainterval',action='store',       dest='ainterval',type='int', default=0, help="alock interval (min).")

    parser.add_option('-C', '--chuser',   action='store',       dest='chuser',  type='string', help="change user in USER_AND_PASS_DIC in niconicodl_cfg.py")

    parser.add_option('-g', '--debug',    action='store_true',  dest='debug',   default=False, help="debug mode (html save to datefilename_?.html)")
    parser.add_option('-t', '--test',     action='store_true',  dest='test',    default=False, help="test mode (not download file (only logging))")

    return parser.parse_args()

#
# nico http server
#
class NicoAppSrv(object):
    index_html = """
        <html><head><title>niconicodl.py %s</title></head><body>
        <p>niconicodl http server mode. now queueing(%d)</p>
        <form action="%s" method="post"><input size=100 type="textbox" name="uri"/><br>
        <input type="checkbox" name="df" value="flv">movie</input>, 
        <input type="checkbox" name="df" value="xml">comment</input>, 
        <input type="checkbox" name="df" value="html">html</input>
        <input type="submit" value="download"/>
        </form>
        <p><a href="%s">update</a></p>
         </body></html>
        """

#    def index(self):
#        return self.index_html % (self.q_len(), niconfig.LOCALHOST)
#    index.exposed = True

    def index(self, uri=False, df=False):
        if not uri:
            uri = False
        if not df:
            df = False
        logging.debug(vars())
        logging.info("Try download %(uri)s" % vars())
        if df and len(df):
            self.kind_pusher(df)
        if uri and len(uri):
            self.uri_pusher(uri)
        # cptools.redirect("./")
        return self.index_html % (__version__, self.q_len(), niconfig.LOCALHOST, niconfig.LOCALHOST)
    index.exposed = True

#
# queue pollinger for http server mode
#
class Pollinger(threading.Thread):
    alive = True
    uri_queue = []
    kind_list = []

    @classmethod
    def uri_pusher(cls,uri):
        cls.uri_queue.append(uri)
        return

    @classmethod
    def kind_pusher(cls,kind):
        if type(kind) == list:
            cls.kind_list.extend(list)
        elif type(kind) == str:
            cls.kind_list.append(kind)
        else:
            pass
        return

    @classmethod
    def kill(cls):
        cls.alive = False
        return

    @classmethod
    def q_len(cls):
        return len(cls.uri_queue)

    def run(self):
        logging.info("start polling.")
        while self.alive:
            time.sleep(niconfig.POLLING_COUNT)
            if len(self.uri_queue):
                logging.info("found q in polling.")
                if len(self.kind_list):
                    nico.retrieve_item = self.kind_list
                nico.run(self.uri_queue)
                self.uri_queue = []
                self.kind_list = []
        return

#
# global start
#
if __name__ == '__main__':

    signal.signal(signal.SIGINT, signal_handler)

    (opts, args) = makeopts()

    nico = NicoDownloader()
    nico.setopt(opts)

    #
    # http server mode when success import cherrypy
    #
    if nico.http_server_mode:
        try:
            import cherrypy
            from cherrypy.lib import cptools
        except ImportError, e:
            logging.exception("import cherrpy. httpservermode error")
            print("import cherrpy exception. httpservermode error")
            sys.exit(1)

        print("http server mode [%s]" % niconfig.LOCALHOST)
        logging.info("http server mode")
        cherrypy.config.update({'environment': 'production',
                                'log.error_file': 'site.log',
                                'server.socket_host': niconfig.HOST,
                                'server.socket_port': niconfig.PORT,
                                'engine.autoreload_on': True,
                                })
        # Mount each app and pass it its own config
        cherrypy.tree.mount(NicoAppSrv, "/")
        NicoAppSrv.uri_pusher = Pollinger.uri_pusher
        NicoAppSrv.kind_pusher = Pollinger.kind_pusher
        NicoAppSrv.q_len = Pollinger.q_len
        NicoAppSrv.nico = nico
        pollinger = Pollinger()
        pollinger.nico = nico
        pollinger.start()
        cherrypy.quickstart(NicoAppSrv())

    else:
        for n in xrange(0, niconfig.LOGIN_RETRY):
            res = nico.run(args)
            if res:
                break
            else:
                print("login failed... try %d time / %d after sleep(%d minutes)" % (n+1, niconfig.LOGIN_RETRY, niconfig.LOGIN_INTERVAL))
                time.sleep(niconfig.LOGIN_INTERVAL * 60)
                print("try again")

##