o 3gk}@s"ddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z ddlmZddlmZddlmZddlmZmZmZmZmZddlmZmZddlZejdd d ZdddZ dddZ!dddZ"dZ#dZ$dddZ%dddZ&ddd Z'dd#d$Z(dd%d&Z)dd(d)Z*d*d+l+m,Z,m-Z-d,d-dd0d1Z.dd7d8Z/dd9d:Z0dd;d<Z1dd?d@Z2ddBdCZ3d,d,dDddHdIZ4dddNdOZ5dPdQddTdUZ6ddWdXZ7dYd,d,dZddmdnZ8e9ej:dodPd,dpdqddsdtZ;ejd*duddvdwZ<dd|d}Z=e;j>d~ddej:dd,ddej:dd3dPddej:dddPddej:ddddPddej?ddd,e=ddddZ@e;j9ddddddZAeAj>ddddddZBeAj>ddddddZCe;j>d=ddej:dd3dPdddddZDe;j9dAdddddZEeEj>dddej?de=ddPddddZFeEj>dddej:ddPddej:ddPddej:dddPddej?de=ddPddddZGe;j>dddej:dddYeHgdddej:ddd,dPddej:dddeIddej:dddeHgdƢddej:dddeIddej:dddeIddej:dddeIddej:dddeIddej:dd,ddej:dddeJddej:dd,dPddej:dd,dPddej:dd,dPddej:dd,dPddej:dd,dPddej?dddPe=ddddZKdddZLeMdkre;dddSdS() annotationsN)IterableSequence) ExitStack)chain)Path)PIPECompletedProcessPopen check_callrun)AnyCallablereturnSequence[str] | NonecCsHzddl}Wn tyYnwtjddgStdrdgStddS)Nr-mmypyz|mypy not found, so can't check config with it. See https://github.com/python/mypy#readme if you want to install it and retry)r ImportErrorsys executableshutilwhichwarning)rr*/home/garg/my-data/HPI/my/core/__main__.pymypy_cmds    rcfg_pathrCompletedProcess | NonecCsitj}|d}t||durdnd|}||d<t}|dur&dStg|dddddd d d tt|d }|S) NMYPYPATH:z--namespace-packagesz--color-outputz--prettyz--show-error-codesz--show-error-contextz--check-untyped-defsz-pz my.config)stderrstdoutenv)osenvirongetstrrr r)rr#mpathcmdmresrrrrun_mypy&s:    r+xr'NonecCstj|dddS)NT)err)clickechor,rrreprintAsr2cCsddd|jddDS)Nrcs|]}d|VqdS)z Nr).0lrrr Hzindent..Tkeependsjoin splitlinesr1rrrindentFsr=u✅u🔲cCsttd|dS)N )r2OKr1rrrinfoOr@cCtd|dS)Nu❌ r2r1rrrerrorSrDcCrB)Nu❗ rCr1rrrrWrEre ExceptioncCs*dtt||j}tjt|dS)Nr) r; tracebackformat_exceptionrG __traceback__rr!writer=)rFtbrrrrL[srLcCsddlm}|}d}|s/|ddd}|jjdd|d td |d}ntd |d t }|r>|sEt ddSdS) N get_mycfg_dirFmyconfigz __init__.pyT)parentsa ### HPI personal config ## see # https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules # https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org ## for some help on writing your own config # to quickly check your config, run: # hpi config check # to quickly check a specific module setup, run hpi doctor , e.g.: # hpi doctor my.reddit.rexport ### useful default imports from my.core import Paths, PathIsh, get_files ### # most of your configs will look like this: class example: export_path: Paths = '/home/user/data/example_data_dir/' ### you can insert your own configuration below ### but feel free to delete the stuff above if you don't need ti zcreated empty config: zconfig directory 'z#' already exists, skipping creation) preinitrOexistsparentmkdir write_textlstripr@rD config_okrexit)rO mycfg_dircreated my_config check_passedrrr config_create`s" r_boolc Csg}ddl}zt|j}Wnty,}z||tdt|WYd}~n d}~wwtd|ddlm }|}ddl }z%t t |j jdj}t ||retd|d|tdWnty}z||t|WYd}~n d}~wwtd |tK}t |d }tjd d d t |g} ztj||ddt| tdd| Wnty}z||t|WYd}~nd}~wwWdn1swYt|} | dur| j} | dkrtdntd|tdtjt| jdtjt| j dzddl!}Wn ty>}z||tdt|WYd}~nd}~wwt"|dkrRtdt"|ddStddS)Nrz&failed to determine module import pathzimport order: rMrNz% Seems that the stub config is used (z). This is likely not going to work. See https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org#setting-up-modules for more information zbad config pathzconfig path : cfgr compileall-qT) dirs_exist_okzsyntax check: r>zmypy check : successzmypy check: failedz mypy failedutf8zfailed to import the configzconfig check: z errorsFzconfig check: success!)#rPlist__path__rGappendrDrLr@rSrOmy.corer'rcorerU startswithstrip RuntimeErrortempfileTemporaryDirectoryrrrcopytreer r;r+ returncoder!rKr=decoder" my.configlen) errorsrPpathsrFrOr core_pkg_pathtdtdirr)mypy_resrcrrrrYs          rYrM) HPIModulemodulesFallrIterable[HPIModule]ccsbg}tD]}|s|jdur||jq|Vqt|dkr/tdt|d|ddSdS)NrzSkipped z modules: z%. Pass --all if you want to see them.)r} skip_reasonrhnamertr)rskippedmrrr_moduless  rverboselist_allquick for_modules list[str]c s^t|dkrd}|r dnd}tddl}ddlm}ddlm}m}ddlm t|dkr5t |d } n fd d |D} | D]} | j } | j } | durbt td tjd ddd| dd | q@zt| } Wn5ty}z)ttjdddd| dd|t|ttfr|||rt|WYd}~q@d}~wwttjdddd| d|| dd}|durt dq@|r|n|}z5i}t|rdt|jvr||d<||d i|}Wdn1swY|dusJdWn)ty$}ztdtjdddd||rt|WYd}~q@d}~wwtd|q@dS)!NrTrz*; pass --verbose to print more informationrM)warn_my_config_import_error) get_stats quick_statsr|r~csg|]}|ddqS)N)rrrr4rrrr z!modules_check..r>SKIPyellowfgz: z<50FAILredz loading failedr?greenz : )guessz2 - no 'stats' function, can't check the datarzstats() returned Nonez - zstats:z& computing failedz - stats: r)rttabulate_warnings contextlibrDrstatsrrutilr|rrrr2OFFr/style importlib import_modulerG isinstancerAttributeErrorrLr@ nullcontextcallableinspect signature parametersr)rrrrvwrrrrmodsmrskiprmodrFr quick_contextkwargsreseerrr modules_checksd      *$  rcCsrtt|dD].}|j}|j}|durt}d}nt}dtjd|ddd}t|d|d|qdS) Nr~rr>z [disabled: ]rr50) rrrrr?rr/rr0)rrrsrpresufrrr list_modulesAsrcs&ddl}|jdfdd }||_dS)z< Helper to avoid visual noise in hpi modules/doctor rNrr'cs*|i|}ddd|jddDS)Nrcsr3)z Nrr4r,rrrr6\r7z6tabulate_warnings..override..Tr8r:)argsrrorigrroverrideZsz#tabulate_warnings..override)rr')warnings formatwarning)rrrrrrRs rr} Sequence[str]csddlmfdd|D}g}|D]+}|jdurt|j|j}|dur0td|jdq|D] }||vr=||q2q|S)NrMmodule_by_namecsg|]}|qSrr)r4modulerrrresz_requires..zModule z has no REQUIRES specification)discovery_purerlegacyrrequiresrrh)r}rrrreqsrrrr _requiresbs    rrcCsJt|tr|g}ddt|dD}td||D]}t|qdS)NcSsg|]}d|dqS)'rrrrrryrz#module_requires..r}zdependencies of )rr'rr2r/r0)rrsr,rrrmodule_requiresus  r)parallelbreak_system_packagesuserrrc CsRt|tr|g}t|}t|dkrtddSdtjv}tjdg|r&dgngdd|r1dgng|r8d gng}g}|rStj d vrS|D] }| g||qFn|rYtd | |t |t :} g} |D]} t d d ttj| | t| } | | qj| D]} | } | dksJ| qWddS1swYdS)Nrz7requirements list is empty, no need to install anythingHPI_MODULE_INSTALL_USE_UVruvpipinstall--user--break-system-packages)win32cygwindarwinzNparallel install is not supported on this platform, installing sequentially...z Running: r>)rr'rrtrr$r%rrplatformrhrfrr2r;mapshlexquote enter_contextr wait)rrrr requirementsuse_uvpre_cmdcmdsr exit_stackpopensr)popenretrrrmodule_installsL       "r Select from: choicespromptintcCst|dks Jdt|dtdd}i}t|dD]"\}}|dkr)t|nt||}|d||<td|d|qtd  t}||vr[t|d t| qD||S) zh Basic menu allowing the user to select one of the choices returns the index the user chose rz$Didnt receive any choices to prompt! a rM z. rTz not in ) rtr2ord enumerater'chrr/getcharrfkeys)rr chr_offset result_mapioptcharchrrr_ui_getchar_picks   rT)rqualified_namesIterable[Callable[..., Any]]c #svddlm}m}ddlmtjsd}|D]}z||VWq|y}zzt |}Wnt yG}z t d|d||d}~wwfddt |t jD}t|d krft d |d |t|dkrr|d Vn.rz7During fallback, could not find any data providers in 'rcSg|]}|jqSr)__name__r4rrrrrz]During fallback, more than one possible data provider, can't prompt since STDOUT is not a TTYzSpecify one of:r.z$Which function should be used from 'z'?z Selected ')queryrrrrrr"isattyrrrGr2r getmembers isfunctionrtr) rrrrqualnameqr_errriedata_providersrfuncname chosen_indexrrr_locate_functions_or_promptsF      rexccCs,ddlm}|ddd}|d|dS)Nr) make_loggerCLIr)levelz hpi query: )rir exception)rrloggerrrr_warn_exceptionss  rjson)outputstreamreverserr order_key str | Noneorder_by_value_type type | Noneafterr beforewithinrlimit int | None drop_unsorted wrap_unsortedwarn_exceptionsraise_exceptionsdrop_exceptionscCsddlm}m}tddt|D}||||||||d|| | | | t| |d }|dkrYddlm}|rN|D]}tj ||tj d q6tj dSt |t|dS|d krxd d lm}|rp|D]}||qgdS|t|dS|d krd dlm}| s|sd} ||tj D]}| rt|q| r||r qtj dSt|}tdt jddddzd dl}Wntytdd dl}|jtdYdSw|dS)NrM) RangeTuple select_rangecss|]}|VqdS)Nrrrrrr6sz&query_hpi_functions..)r$r%r&) r r"unparsed_rangerr'r)r*r+ warn_funcr,r-r)dumpsrpprintr)r3gpx)locations_to_gpxTz( Interact with the results by using the rrrz variable zj'repl' typically uses ipython, install it with 'python3 -m pip install ipython'. falling back to stdlib...)local) query_ranger.r/rrr serializer2rr"rKflushr/r0rfr3my.location.commonr5r2rIPythonModuleNotFoundErrorcodeinteractlocalsembed)rrrr r"r$r%r&rr'r)r*r+r,r-r.r/ input_srcrr2itemr3r5rr;r=rrrquery_hpi_functionssl         rCz--debugzShow debug logs)is_flagdefaulthelpdebugcCs8|rdtjd<ttd}|jddt|dS)z Human Programming Interface Tool for HPI Work in progress, will be used for config management, troubleshooting & introspection rGLOGGING_LEVEL_HPI hpi_temp_dirT)exist_okN)r$r%rrn gettempdirrVchdir)rGryrrrmaincs   rM)maxsizecCstddtDS)zEShould include all modules, in case user is trying to diagnose issuescSrr)rrrrrrrz"_all_mod_names..)sortedr}rrrr_all_mod_namessrPctx click.Contextr incompletecsfddtDS)Ncsg|] }|r|qSr)rkrrSrrrsz(_module_autocomplete..)rP)rQrrSrrTr_module_autocompleterArUdoctorzrun various checks)r short_helpz--verbose/--quietz!Print more diagnostic information)rErFz--allz$List all modules, including disabled)rDrFrcz--quickz)Only run partial checks (first 100 items)z-Sz--skip-config-check skip_confzSkip configuration checkMODULE)nargsrequiredshell_completecCs"|stt|||t|ddS)z Run various checks MODULE is one or more specific module names to check (e.g. my.reddit.rexport) Otherwise, checks all modules )rrrrN)rYrrf)rrrrXrrrr doctor_cmds r^rQzwork with configurationcCdS)zAct on your HPI configurationNrrrrr config_grpr`checkz check configcCs"t}t|r ddSddS)z!Check your HPI configuration filerFN)rYrrZ)okrrrconfig_check_cmdsrdcreatezcreate user configcCs tdS)z&Create user configuration file for HPIN)r_rrrrconfig_create_cmds rfzlist available modulescCt|ddS)zList available modulesrN)rrhrrr module_cmdsrizmodule managementcCr_)zModule managementNrrrrr module_grprarjrzprint module reqsMODULES)r]r[r\cCrg)zo Print MODULES requirements MODULES is one or more specific module names (e.g. my.reddit.rexport) )rN)rrrrrmodule_requires_cmdsrlrzinstall module depsrzsame as pip --userz --parallelz/EXPERIMENTAL. Install dependencies in parallel.z-BrzVBypass PEP 668 and install dependencies into the system-wide python package directory.cCst||||ddS)z Install dependencies for modules using pip MODULES is one or more specific module names (e.g. my.reddit.rexport) )rrrrN)r)rrrr}rrrmodule_install_cmdsrmr z#query the results of a HPI functionz-oz--output)rr3replr4z*what to do with the result [default: json])rEtyperFz-sz--streamzIstream objects from the data source instead of printing a list at the end)rErDrFz-kz --order-keyz_order by an object attribute or dict key on the individual objects returned by the HPI functionz-tz --order-type)datetimedaterfloatz0order by searching for some type on the iterablez-az--afterzMwhile ordering, filter items for the key or type larger than or equal to thisz-bz--beforezBwhile ordering, filter items for the key or type smaller than thisz-wz--withinzQa range 'after' or 'before' to filter items by. see above for further explanationz-rz--recentzYa shorthand for '--order-type datetime --reverse --before now --within'. e.g. --recent 5dz--reverse/--no-reversez/reverse the results returned from the functionsz-lz--limitz7limit the number of items returned from the (functions)z--drop-unsortedz]if the order of an item can't be determined while ordering, drop those items from the resultsz--wrap-unsortedzaif the order of an item can't be determined while ordering, wrap them into an 'Unsortable' objectz--warn-exceptionsz:if any errors are returned, print them as errors on STDERRz--raise-exceptionszRif any errors are returned (as objects, not raised) from the functions, raise themz--drop-exceptionsz8ignore any errors returned as objects from the functions FUNCTION_NAME function_name order_typerecentcCsddlm}m}|dkr|}n|dkr|}n|dkrt}n |dkr$t}nd}|dur5d}|p/|}|}| } d d lm}zt||t||||||| | | | | ||d WdS|yq}ztt |t d WYd}~dSd}~ww) a| This allows you to query the results from one or more functions in HPI By default this runs with '-o json', converting the results to JSON and printing them to STDOUT You can specify '-o pprint' to just print the objects using their repr, or '-o repl' to drop into a ipython shell with access to the results While filtering using --order-key datetime, the --after, --before and --within flags parse the input to their datetime and timedelta equivalents. datetimes can be epoch time, the string 'now', or an date formatted in the ISO format. timedelta (durations) are parsed from a similar format to the GNU 'sleep' command, e.g. 1w2d8h5m20s -> 1 week, 2 days, 8 hours, 5 minutes, 20 seconds As an example, to query reddit comments I've made in the last month  hpi query --order-type datetime --before now --within 4w my.reddit.all.comments or... hpi query --recent 4w my.reddit.all.comments  Can also query within a range. To filter comments between 2016 and 2018: hpi query --order-type datetime --after '2016-01-01' --before '2019-01-01' my.reddit.all.comments r)rqrprprqrrrNnowrM)r)rrrr r"r$r%r&rr'r)r*r+r,r-) rprqrrrr rrCrfr2r'rrZ)rtrrr rur$r%r&rvrr'r)r*r+r,r-rqrpchosen_order_typerqerrr query_cmdsNt   rzcCsLddlm}|tgd}|jdksJd|jvsJd|jvs$JdS)Nr) CliRunner)rrzmy.github.ghexportzmy.browser.exportzgithub.com/karlicoss/ghexport browserexport) click.testingr{invokerM exit_coder)r{resultrrr test_requiress r__main__hpi) prog_name)rr)rrrr)r,r'rr-)r,r'rr')rFrGrr-)rr-)rr`)rr`rr) rr`rr`rr`rrrr-)rr`rr-)r}rrr)rrrr-) rr`rrrr`rr`rr-)r)rrrr'rr)rrrr`rr)rrGrr-) rr'rr`rrr r!r"r#r$r r%r r&r rr`r'r(r)r`r*r`r+r`r,r`r-r`rr-)rGr`rr-)rr)rQrRrrrSr'rr) rr`rr`rr`rXr`rrrr-)r}rrr-) rr`rr`rr`r}rrr-)"rtrrr'rr`r r!rur!r$r!r%r!r&r!rvr!rr`r'r(r)r`r*r`r+r`r,r`r-r`rr-)N __future__r functoolsrrr$rrrrnrHcollections.abcrrrr itertoolsrpathlibr subprocessrr r r r typingr rr/ lru_cacherr+r2r=r?rr@rDrrLr_rYrr|r}rrrrrrrrrrrCgroupoptionrMrPrUcommandargumentr^r`rdrfrirjrlrmChoicer'rrzrrrrrrsX              0^ E    2 0 ]                . b