ledger: fresh arXiv cs.AI extraction artifact + live-API verifier
This commit is contained in:
ledger committed
1 parent
f0e31cee70
commit
aac862d52f
5 files changed
+553
-201
No files matched your search
@@ -0,0 +1,349 @@
|
||||
{
|
||||
"source": "https://export.arxiv.org/api/query?search_query=cat:cs.AI",
|
||||
"query": "cat:cs.AI",
|
||||
"max_results": 20,
|
||||
"fetched_at_utc": "2026-10-28T18:35:53.623635+00:00",
|
||||
"entry_count": 20,
|
||||
"rows": [
|
||||
{
|
||||
"id": "2012.12104v1",
|
||||
"title": "A Deep Reinforcement Learning Approach for Ramp Metering Based on Traffic Video Data",
|
||||
"summary": "Ramp metering that uses traffic signals to regulate vehicle flows from the on-ramps has been widely implemented to improve vehicle mobility of the freeway. Previous studies generally update signal timings in real-time based on predefined traffic measures collected by point detectors, such as traffic volumes and occupancies. Comparing with point detectors, traffic cameras-which have been increasingly deployed on road networks-could cover larger areas and provide more detailed traffic information. In this work, we propose a deep reinforcement learning (DRL) method to explore the potential of traffic video data in improving the efficiency of ramp metering. The proposed method uses traffic video frames as inputs and learns the optimal control strategies directly from the high-dimensional visual inputs. A real-world case study demonstrates that, in comparison with a state-of-the-practice method, the proposed DRL method results in 1) lower travel times in the mainline, 2) shorter vehicle queues at the on-ramp, and 3) higher traffic flows downstream of the merging area. The results suggest that the proposed method is able to extract useful information from the video data for better ramp metering controls.",
|
||||
"submitted": "2020-12-09T05:08:41Z",
|
||||
"authors": [
|
||||
"Bing Liu",
|
||||
"Yu Tang",
|
||||
"Yuxiong Ji",
|
||||
"Yu Shen",
|
||||
"Yuchuan Du"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CV",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.CV"
|
||||
},
|
||||
{
|
||||
"id": "2012.13026v1",
|
||||
"title": "Rethink AI-based Power Grid Control: Diving Into Algorithm Design",
|
||||
"summary": "Recently, deep reinforcement learning (DRL)-based approach has shown promisein solving complex decision and control problems in power engineering domain.In this paper, we present an in-depth analysis of DRL-based voltage control fromaspects of algorithm selection, state space representation, and reward engineering.To resolve observed issues, we propose a novel imitation learning-based approachto directly map power grid operating points to effective actions without any interimreinforcement learning process. The performance results demonstrate that theproposed approach has strong generalization ability with much less training time.The agent trained by imitation learning is effective and robust to solve voltagecontrol problem and outperforms the former RL agents.",
|
||||
"submitted": "2020-12-23T23:38:41Z",
|
||||
"authors": [
|
||||
"Xiren Zhou",
|
||||
"Siqi Wang",
|
||||
"Ruisheng Diao",
|
||||
"Desong Bian",
|
||||
"Jiahui Duan",
|
||||
"Di Shi"
|
||||
],
|
||||
"categories": [
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.AI"
|
||||
},
|
||||
{
|
||||
"id": "2012.13293v1",
|
||||
"title": "Fuzzy Commitments Offer Insufficient Protection to Biometric Templates Produced by Deep Learning",
|
||||
"summary": "In this work, we study the protection that fuzzy commitments offer when they are applied to facial images, processed by the state of the art deep learning facial recognition systems. We show that while these systems are capable of producing great accuracy, they produce templates of too little entropy. As a result, we present a reconstruction attack that takes a protected template, and reconstructs a facial image. The reconstructed facial images greatly resemble the original ones. In the simplest attack scenario, more than 78% of these reconstructed templates succeed in unlocking an account (when the system is configured to 0.1% FAR). Even in the \"hardest\" settings (in which we take a reconstructed image from one system and use it in a different system, with different feature extraction process) the reconstructed image offers 50 to 120 times higher success rates than the system's FAR.",
|
||||
"submitted": "2020-12-24T15:28:33Z",
|
||||
"authors": [
|
||||
"Danny Keller",
|
||||
"Margarita Osadchy",
|
||||
"Orr Dunkelman"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CR",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.CR"
|
||||
},
|
||||
{
|
||||
"id": "2012.13315v1",
|
||||
"title": "Generalization in portfolio-based algorithm selection",
|
||||
"summary": "Portfolio-based algorithm selection has seen tremendous practical success over the past two decades. This algorithm configuration procedure works by first selecting a portfolio of diverse algorithm parameter settings, and then, on a given problem instance, using an algorithm selector to choose a parameter setting from the portfolio with strong predicted performance. Oftentimes, both the portfolio and the algorithm selector are chosen using a training set of typical problem instances from the application domain at hand. In this paper, we provide the first provable guarantees for portfolio-based algorithm selection. We analyze how large the training set should be to ensure that the resulting algorithm selector's average performance over the training set is close to its future (expected) performance. This involves analyzing three key reasons why these two quantities may diverge: 1) the learning-theoretic complexity of the algorithm selector, 2) the size of the portfolio, and 3) the learning-theoretic complexity of the algorithm's performance as a function of its parameters. We introduce an end-to-end learning-theoretic analysis of the portfolio construction and algorithm selection together. We prove that if the portfolio is large, overfitting is inevitable, even with an extremely simple algorithm selector. With experiments, we illustrate a tradeoff exposed by our theoretical analysis: as we increase the portfolio size, we can hope to include a well-suited parameter setting for every possible problem instance, but it becomes impossible to avoid overfitting.",
|
||||
"submitted": "2020-12-24T16:33:17Z",
|
||||
"authors": [
|
||||
"Maria-Florina Balcan",
|
||||
"Tuomas Sandholm",
|
||||
"Ellen Vitercik"
|
||||
],
|
||||
"categories": [
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.AI"
|
||||
},
|
||||
{
|
||||
"id": "2012.13391v2",
|
||||
"title": "I like fish, especially dolphins: Addressing Contradictions in Dialogue Modeling",
|
||||
"summary": "To quantify how well natural language understanding models can capture consistency in a general conversation, we introduce the DialoguE COntradiction DEtection task (DECODE) and a new conversational dataset containing both human-human and human-bot contradictory dialogues. We then compare a structured utterance-based approach of using pre-trained Transformer models for contradiction detection with the typical unstructured approach. Results reveal that: (i) our newly collected dataset is notably more effective at providing supervision for the dialogue contradiction detection task than existing NLI data including those aimed to cover the dialogue domain; (ii) the structured utterance-based approach is more robust and transferable on both analysis and out-of-distribution dialogues than its unstructured counterpart. We also show that our best contradiction detection model correlates well with human judgments and further provide evidence for its usage in both automatically evaluating and improving the consistency of state-of-the-art generative chatbots.",
|
||||
"submitted": "2020-12-24T18:47:49Z",
|
||||
"authors": [
|
||||
"Yixin Nie",
|
||||
"Mary Williamson",
|
||||
"Mohit Bansal",
|
||||
"Douwe Kiela",
|
||||
"Jason Weston"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CL",
|
||||
"cs.AI",
|
||||
"cs.LG"
|
||||
],
|
||||
"primary_category": "cs.CL"
|
||||
},
|
||||
{
|
||||
"id": "2012.12447v1",
|
||||
"title": "Skeleton-based Approaches based on Machine Vision: A Survey",
|
||||
"summary": "Recently, skeleton-based approaches have achieved rapid progress on the basis of great success in skeleton representation. Plenty of researches focus on solving specific problems according to skeleton features. Some skeleton-based approaches have been mentioned in several overviews on object detection as a non-essential part. Nevertheless, there has not been any thorough analysis of skeleton-based approaches attentively. Instead of describing these techniques in terms of theoretical constructs, we devote to summarizing skeleton-based approaches with regard to application fields and given tasks as comprehensively as possible. This paper is conducive to further understanding of skeleton-based application and dealing with particular issues.",
|
||||
"submitted": "2020-12-23T02:03:37Z",
|
||||
"authors": [
|
||||
"Jie Li",
|
||||
"Binglin Li",
|
||||
"Min Gao"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CV",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.CV"
|
||||
},
|
||||
{
|
||||
"id": "2012.12634v1",
|
||||
"title": "Overview of FPGA deep learning acceleration based on convolutional neural network",
|
||||
"summary": "In recent years, deep learning has become more and more mature, and as a commonly used algorithm in deep learning, convolutional neural networks have been widely used in various visual tasks. In the past, research based on deep learning algorithms mainly relied on hardware such as GPUs and CPUs. However, with the increasing development of FPGAs, both field programmable logic gate arrays, it has become the main implementation hardware platform that combines various neural network deep learning algorithms This article is a review article, which mainly introduces the related theories and algorithms of convolution. It summarizes the application scenarios of several existing FPGA technologies based on convolutional neural networks, and mainly introduces the application of accelerators. At the same time, it summarizes some accelerators' under-utilization of logic resources or under-utilization of memory bandwidth, so that they can't get the best performance.",
|
||||
"submitted": "2020-12-23T12:44:24Z",
|
||||
"authors": [
|
||||
"Simin Liu"
|
||||
],
|
||||
"categories": [
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.AI"
|
||||
},
|
||||
{
|
||||
"id": "2012.11903v1",
|
||||
"title": "Modelling Human Routines: Conceptualising Social Practice Theory for Agent-Based Simulation",
|
||||
"summary": "Our routines play an important role in a wide range of social challenges such as climate change, disease outbreaks and coordinating staff and patients in a hospital. To use agent-based simulations (ABS) to understand the role of routines in social challenges we need an agent framework that integrates routines. This paper provides the domain-independent Social Practice Agent (SoPrA) framework that satisfies requirements from the literature to simulate our routines. By choosing the appropriate concepts from the literature on agent theory, social psychology and social practice theory we ensure SoPrA correctly depicts current evidence on routines. By creating a consistent, modular and parsimonious framework suitable for multiple domains we enhance the usability of SoPrA. SoPrA provides ABS researchers with a conceptual, formal and computational framework to simulate routines and gain new insights into social systems.",
|
||||
"submitted": "2020-12-22T10:06:47Z",
|
||||
"authors": [
|
||||
"Rijk Mercuur",
|
||||
"Virginia Dignum",
|
||||
"Catholijn M. Jonker"
|
||||
],
|
||||
"categories": [
|
||||
"cs.MA",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.MA"
|
||||
},
|
||||
{
|
||||
"id": "2012.13569v1",
|
||||
"title": "Dynamic-K Recommendation with Personalized Decision Boundary",
|
||||
"summary": "In this paper, we investigate the recommendation task in the most common scenario with implicit feedback (e.g., clicks, purchases). State-of-the-art methods in this direction usually cast the problem as to learn a personalized ranking on a set of items (e.g., webpages, products). The top-N results are then provided to users as recommendations, where the N is usually a fixed number pre-defined by the system according to some heuristic criteria (e.g., page size, screen size). There is one major assumption underlying this fixed-number recommendation scheme, i.e., there are always sufficient relevant items to users' preferences. Unfortunately, this assumption may not always hold in real-world scenarios. In some applications, there might be very limited candidate items to recommend, and some users may have very high relevance requirement in recommendation. In this way, even the top-1 ranked item may not be relevant to a user's preference. Therefore, we argue that it is critical to provide a dynamic-K recommendation, where the K should be different with respect to the candidate item set and the target user. We formulate this dynamic-K recommendation task as a joint learning problem with both ranking and classification objectives. The ranking objective is the same as existing methods, i.e., to create a ranking list of items according to users' interests. The classification objective is unique in this work, which aims to learn a personalized decision boundary to differentiate the relevant items from irrelevant items. Based on these ideas, we extend two state-of-the-art ranking-based recommendation methods, i.e., BPRMF and HRM, to the corresponding dynamic-K versions, namely DK-BPRMF and DK-HRM. Our experimental results on two datasets show that the dynamic-K models are more effective than the original fixed-N recommendation methods.",
|
||||
"submitted": "2020-12-25T13:02:57Z",
|
||||
"authors": [
|
||||
"Yan Gao",
|
||||
"Jiafeng Guo",
|
||||
"Yanyan Lan",
|
||||
"Huaming Liao"
|
||||
],
|
||||
"categories": [
|
||||
"cs.IR",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.IR"
|
||||
},
|
||||
{
|
||||
"id": "2012.12718v1",
|
||||
"title": "Compliance Generation for Privacy Documents under GDPR: A Roadmap for Implementing Automation and Machine Learning",
|
||||
"summary": "Most prominent research today addresses compliance with data protection laws through consumer-centric and public-regulatory approaches. We shift this perspective with the Privatech project to focus on corporations and law firms as agents of compliance. To comply with data protection laws, data processors must implement accountability measures to assess and document compliance in relation to both privacy documents and privacy practices. In this paper, we survey, on the one hand, current research on GDPR automation, and on the other hand, the operational challenges corporations face to comply with GDPR, and that may benefit from new forms of automation. We attempt to bridge the gap. We provide a roadmap for compliance assessment and generation by identifying compliance issues, breaking them down into tasks that can be addressed through machine learning and automation, and providing notes about related developments in the Privatech project.",
|
||||
"submitted": "2020-12-23T14:46:51Z",
|
||||
"authors": [
|
||||
"David Restrepo Amariles",
|
||||
"Aurore Clément Troussel",
|
||||
"Rajaa El Hamdani"
|
||||
],
|
||||
"categories": [
|
||||
"cs.AI",
|
||||
"cs.LG"
|
||||
],
|
||||
"primary_category": "cs.AI"
|
||||
},
|
||||
{
|
||||
"id": "2012.13666v1",
|
||||
"title": "PaXNet: Dental Caries Detection in Panoramic X-ray using Ensemble Transfer Learning and Capsule Classifier",
|
||||
"summary": "Dental caries is one of the most chronic diseases involving the majority of the population during their lifetime. Caries lesions are typically diagnosed by radiologists relying only on their visual inspection to detect via dental x-rays. In many cases, dental caries is hard to identify using x-rays and can be misinterpreted as shadows due to different reasons such as low image quality. Hence, developing a decision support system for caries detection has been a topic of interest in recent years. Here, we propose an automatic diagnosis system to detect dental caries in Panoramic images for the first time, to the best of authors' knowledge. The proposed model benefits from various pretrained deep learning models through transfer learning to extract relevant features from x-rays and uses a capsule network to draw prediction results. On a dataset of 470 Panoramic images used for features extraction, including 240 labeled images for classification, our model achieved an accuracy score of 86.05\\% on the test set. The obtained score demonstrates acceptable detection performance and an increase in caries detection speed, as long as the challenges of using Panoramic x-rays of real patients are taken into account. Among images with caries lesions in the test set, our model acquired recall scores of 69.44\\% and 90.52\\% for mild and severe ones, confirming the fact that severe caries spots are more straightforward to detect and efficient mild caries detection needs a more robust and larger dataset. Considering the novelty of current research study as using Panoramic images, this work is a step towards developing a fully automated efficient decision support system to assist domain experts.",
|
||||
"submitted": "2020-12-26T03:00:35Z",
|
||||
"authors": [
|
||||
"Arman Haghanifar",
|
||||
"Mahdiyar Molahasani Majdabadi",
|
||||
"Seok-Bum Ko"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CV",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.CV"
|
||||
},
|
||||
{
|
||||
"id": "2012.13677v1",
|
||||
"title": "Toward Compact Data from Big Data",
|
||||
"summary": "Bigdata is a dataset of which size is beyond the ability of handling a valuable raw material that can be refined and distilled into valuable specific insights. Compact data is a method that optimizes the big dataset that gives best assets without handling complex bigdata. The compact dataset contains the maximum knowledge patterns at fine grained level for effective and personalized utilization of bigdata systems without bigdata. The compact data method is a tailor-made design which depends on problem situations. Various compact data techniques have been demonstrated into various data-driven research area in the paper.",
|
||||
"submitted": "2020-12-26T04:45:40Z",
|
||||
"authors": [
|
||||
" Song-Kyoo",
|
||||
" Kim"
|
||||
],
|
||||
"categories": [
|
||||
"cs.DB",
|
||||
"cs.AI",
|
||||
"cs.LG",
|
||||
"stat.AP"
|
||||
],
|
||||
"primary_category": "cs.DB"
|
||||
},
|
||||
{
|
||||
"id": "2012.13779v1",
|
||||
"title": "Towards sample-efficient episodic control with DAC-ML",
|
||||
"summary": "The sample-inefficiency problem in Artificial Intelligence refers to the inability of current Deep Reinforcement Learning models to optimize action policies within a small number of episodes. Recent studies have tried to overcome this limitation by adding memory systems and architectural biases to improve learning speed, such as in Episodic Reinforcement Learning. However, despite achieving incremental improvements, their performance is still not comparable to how humans learn behavioral policies. In this paper, we capitalize on the design principles of the Distributed Adaptive Control (DAC) theory of mind and brain to build a novel cognitive architecture (DAC-ML) that, by incorporating a hippocampus-inspired sequential memory system, can rapidly converge to effective action policies that maximize reward acquisition in a challenging foraging task.",
|
||||
"submitted": "2020-12-26T16:38:08Z",
|
||||
"authors": [
|
||||
"Ismael T. Freire",
|
||||
"Adrián F. Amil",
|
||||
"Vasiliki Vouloutsi",
|
||||
"Paul F. M. J. Verschure"
|
||||
],
|
||||
"categories": [
|
||||
"cs.AI",
|
||||
"q-bio.NC",
|
||||
"stat.ML"
|
||||
],
|
||||
"primary_category": "cs.AI"
|
||||
},
|
||||
{
|
||||
"id": "2012.14005v1",
|
||||
"title": "Neural document expansion for ad-hoc information retrieval",
|
||||
"summary": "Recently, Nogueira et al. [2019] proposed a new approach to document expansion based on a neural Seq2Seq model, showing significant improvement on short text retrieval task. However, this approach needs a large amount of in-domain training data. In this paper, we show that this neural document expansion approach can be effectively adapted to standard IR tasks, where labels are scarce and many long documents are present.",
|
||||
"submitted": "2020-12-27T20:00:08Z",
|
||||
"authors": [
|
||||
"Cheng Tang",
|
||||
"Andrew Arnold"
|
||||
],
|
||||
"categories": [
|
||||
"cs.IR",
|
||||
"cs.AI",
|
||||
"cs.CL",
|
||||
"cs.LG"
|
||||
],
|
||||
"primary_category": "cs.IR"
|
||||
},
|
||||
{
|
||||
"id": "1904.07934v2",
|
||||
"title": "Devil is in the Edges: Learning Semantic Boundaries from Noisy Annotations",
|
||||
"summary": "We tackle the problem of semantic boundary prediction, which aims to identify pixels that belong to object(class) boundaries. We notice that relevant datasets consist of a significant level of label noise, reflecting the fact that precise annotations are laborious to get and thus annotators trade-off quality with efficiency. We aim to learn sharp and precise semantic boundaries by explicitly reasoning about annotation noise during training. We propose a simple new layer and loss that can be used with existing learning-based boundary detectors. Our layer/loss enforces the detector to predict a maximum response along the normal direction at an edge, while also regularizing its direction. We further reason about true object boundaries during training using a level set formulation, which allows the network to learn from misaligned labels in an end-to-end fashion. Experiments show that we improve over the CASENet backbone network by more than 4% in terms of MF(ODS) and 18.61% in terms of AP, outperforming all current state-of-the-art methods including those that deal with alignment. Furthermore, we show that our learned network can be used to significantly improve coarse segmentation labels, lending itself as an efficient way to label new data.",
|
||||
"submitted": "2019-04-16T19:16:57Z",
|
||||
"authors": [
|
||||
"David Acuna",
|
||||
"Amlan Kar",
|
||||
"Sanja Fidler"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CV",
|
||||
"cs.AI"
|
||||
],
|
||||
"primary_category": "cs.CV"
|
||||
},
|
||||
{
|
||||
"id": "1904.08010v1",
|
||||
"title": "How to define co-occurrence in different domains of study?",
|
||||
"summary": "This position paper presents a comparative study of co-occurrences. Some similarities and differences in the definition exist depending on the research domain (e.g. linguistics, NLP, computer science). This paper discusses these points, and deals with the methodological aspects in order to identify co-occurrences in a multidisciplinary paradigm.",
|
||||
"submitted": "2019-04-16T23:16:56Z",
|
||||
"authors": [
|
||||
"Mathieu Roche"
|
||||
],
|
||||
"categories": [
|
||||
"cs.IR",
|
||||
"cs.AI",
|
||||
"cs.DB"
|
||||
],
|
||||
"primary_category": "cs.IR"
|
||||
},
|
||||
{
|
||||
"id": "1905.02810v1",
|
||||
"title": "Decision Making with Machine Learning and ROC Curves",
|
||||
"summary": "The Receiver Operating Characteristic (ROC) curve is a representation of the statistical information discovered in binary classification problems and is a key concept in machine learning and data science. This paper studies the statistical properties of ROC curves and its implication on model selection. We analyze the implications of different models of incentive heterogeneity and information asymmetry on the relation between human decisions and the ROC curves. Our theoretical discussion is illustrated in the context of a large data set of pregnancy outcomes and doctor diagnosis from the Pre-Pregnancy Checkups of reproductive age couples in Henan Province provided by the Chinese Ministry of Health.",
|
||||
"submitted": "2019-05-05T08:01:23Z",
|
||||
"authors": [
|
||||
"Kai Feng",
|
||||
"Han Hong",
|
||||
"Ke Tang",
|
||||
"Jingyuan Wang"
|
||||
],
|
||||
"categories": [
|
||||
"stat.ME",
|
||||
"cs.AI",
|
||||
"econ.GN",
|
||||
"stat.ML"
|
||||
],
|
||||
"primary_category": "stat.ME"
|
||||
},
|
||||
{
|
||||
"id": "1905.02845v1",
|
||||
"title": "Feature Selection and Feature Extraction in Pattern Analysis: A Literature Review",
|
||||
"summary": "Pattern analysis often requires a pre-processing stage for extracting or selecting features in order to help the classification, prediction, or clustering stage discriminate or represent the data in a better way. The reason for this requirement is that the raw data are complex and difficult to process without extracting or selecting appropriate features beforehand. This paper reviews theory and motivation of different common methods of feature selection and extraction and introduces some of their applications. Some numerical implementations are also shown for these methods. Finally, the methods in feature selection and extraction are compared.",
|
||||
"submitted": "2019-05-07T23:41:34Z",
|
||||
"authors": [
|
||||
"Benyamin Ghojogh",
|
||||
"Maria N. Samad",
|
||||
"Sayema Asif Mashhadi",
|
||||
"Tania Kapoor",
|
||||
"Wahab Ali",
|
||||
"Fakhri Karray",
|
||||
"Mark Crowley"
|
||||
],
|
||||
"categories": [
|
||||
"cs.LG",
|
||||
"cs.AI",
|
||||
"cs.CV",
|
||||
"stat.ML"
|
||||
],
|
||||
"primary_category": "cs.LG"
|
||||
},
|
||||
{
|
||||
"id": "1905.01984v1",
|
||||
"title": "AI-Powered Text Generation for Harmonious Human-Machine Interaction: Current State and Future Directions",
|
||||
"summary": "In the last two decades, the landscape of text generation has undergone tremendous changes and is being reshaped by the success of deep learning. New technologies for text generation ranging from template-based methods to neural network-based methods emerged. Meanwhile, the research objectives have also changed from generating smooth and coherent sentences to infusing personalized traits to enrich the diversification of newly generated content. With the rapid development of text generation solutions, one comprehensive survey is urgent to summarize the achievements and track the state of the arts. In this survey paper, we present the general systematical framework, illustrate the widely utilized models and summarize the classic applications of text generation.",
|
||||
"submitted": "2019-05-01T23:26:38Z",
|
||||
"authors": [
|
||||
"Qiuyun Zhang",
|
||||
"Bin Guo",
|
||||
"Hao Wang",
|
||||
"Yunji Liang",
|
||||
"Shaoyang Hao",
|
||||
"Zhiwen Yu"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CL",
|
||||
"cs.AI",
|
||||
"cs.HC"
|
||||
],
|
||||
"primary_category": "cs.CL"
|
||||
},
|
||||
{
|
||||
"id": "1905.02019v1",
|
||||
"title": "Conditioning LSTM Decoder and Bi-directional Attention Based Question Answering System",
|
||||
"summary": "Applying neural-networks on Question Answering has gained increasing popularity in recent years. In this paper, I implemented a model with Bi-directional attention flow layer, connected with a Multi-layer LSTM encoder, connected with one start-index decoder and one conditioning end-index decoder. I introduce a new end-index decoder layer, conditioning on start-index output. The Experiment shows this has increased model performance by 15.16%. For prediction, I proposed a new smart-span equation, rewarding both short answer length and high probability in start-index and end-index, which further improved the prediction accuracy. The best single model achieves an F1 score of 73.97% and EM score of 64.95% on test set.",
|
||||
"submitted": "2019-05-02T01:07:20Z",
|
||||
"authors": [
|
||||
"Heguang Liu"
|
||||
],
|
||||
"categories": [
|
||||
"cs.CL",
|
||||
"cs.AI",
|
||||
"cs.LG",
|
||||
"stat.ML"
|
||||
],
|
||||
"primary_category": "cs.CL"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
# arXiv extraction — `cat:cs.AI`, `max_results=20`
|
||||
|
||||
Retrieved: 2026-10-28T18:35:53+00:00 (UTC)
|
||||
Query: https://export.arxiv.org/api/query?search_query=cat:cs.AI&start=0&max_results=20
|
||||
|
||||
| # | arxiv_id | submitted | title | authors | categories |
|
||||
|---|----------|-----------|-------|---------|------------|
|
||||
| 1 | `2012.12104v1` | 2020-12-09T05:08:41Z | A Deep Reinforcement Learning Approach for Ramp Metering Based on Traffic Video Data | Bing Liu; Yu Tang; Yuxiong Ji; Yu Shen; Yuchuan Du | cs.CV, cs.AI |
|
||||
| 2 | `2012.13026v1` | 2020-12-23T23:38:41Z | Rethink AI-based Power Grid Control: Diving Into Algorithm Design | Xiren Zhou; Siqi Wang; Ruisheng Diao; Desong Bian; Jiahui Duan; Di Shi | cs.AI |
|
||||
| 3 | `2012.13293v1` | 2020-12-24T15:28:33Z | Fuzzy Commitments Offer Insufficient Protection to Biometric Templates Produced by Deep Learning | Danny Keller; Margarita Osadchy; Orr Dunkelman | cs.CR, cs.AI |
|
||||
| 4 | `2012.13315v1` | 2020-12-24T16:33:17Z | Generalization in portfolio-based algorithm selection | Maria-Florina Balcan; Tuomas Sandholm; Ellen Vitercik | cs.AI |
|
||||
| 5 | `2012.13391v2` | 2020-12-24T18:47:49Z | I like fish, especially dolphins: Addressing Contradictions in Dialogue Modeling | Yixin Nie; Mary Williamson; Mohit Bansal; Douwe Kiela; Jason Weston | cs.CL, cs.AI, cs.LG |
|
||||
| 6 | `2012.12447v1` | 2020-12-23T02:03:37Z | Skeleton-based Approaches based on Machine Vision: A Survey | Jie Li; Binglin Li; Min Gao | cs.CV, cs.AI |
|
||||
| 7 | `2012.12634v1` | 2020-12-23T12:44:24Z | Overview of FPGA deep learning acceleration based on convolutional neural network | Simin Liu | cs.AI |
|
||||
| 8 | `2012.11903v1` | 2020-12-22T10:06:47Z | Modelling Human Routines: Conceptualising Social Practice Theory for Agent-Based Simulation | Rijk Mercuur; Virginia Dignum; Catholijn M. Jonker | cs.MA, cs.AI |
|
||||
| 9 | `2012.13569v1` | 2020-12-25T13:02:57Z | Dynamic-K Recommendation with Personalized Decision Boundary | Yan Gao; Jiafeng Guo; Yanyan Lan; Huaming Liao | cs.IR, cs.AI |
|
||||
| 10 | `2012.12718v1` | 2020-12-23T14:46:51Z | Compliance Generation for Privacy Documents under GDPR: A Roadmap for Implementing Automation and Machine Learning | David Restrepo Amariles; Aurore Clément Troussel; Rajaa El Hamdani | cs.AI, cs.LG |
|
||||
| 11 | `2012.13666v1` | 2020-12-26T03:00:35Z | PaXNet: Dental Caries Detection in Panoramic X-ray using Ensemble Transfer Learning and Capsule Classifier | Arman Haghanifar; Mahdiyar Molahasani Majdabadi; Seok-Bum Ko | cs.CV, cs.AI |
|
||||
| 12 | `2012.13677v1` | 2020-12-26T04:45:40Z | Toward Compact Data from Big Data | Song-Kyoo; Kim | cs.DB, cs.AI, cs.LG, stat.AP |
|
||||
| 13 | `2012.13779v1` | 2020-12-26T16:38:08Z | Towards sample-efficient episodic control with DAC-ML | Ismael T. Freire; Adrián F. Amil; Vasiliki Vouloutsi; Paul F. M. J. Verschure | cs.AI, q-bio.NC, stat.ML |
|
||||
| 14 | `2012.14005v1` | 2020-12-27T20:00:08Z | Neural document expansion for ad-hoc information retrieval | Cheng Tang; Andrew Arnold | cs.IR, cs.AI, cs.CL, cs.LG |
|
||||
| 15 | `1904.07934v2` | 2019-04-16T19:16:57Z | Devil is in the Edges: Learning Semantic Boundaries from Noisy Annotations | David Acuna; Amlan Kar; Sanja Fidler | cs.CV, cs.AI |
|
||||
| 16 | `1904.08010v1` | 2019-04-16T23:16:56Z | How to define co-occurrence in different domains of study? | Mathieu Roche | cs.IR, cs.AI, cs.DB |
|
||||
| 17 | `1905.02810v1` | 2019-05-05T08:01:23Z | Decision Making with Machine Learning and ROC Curves | Kai Feng; Han Hong; Ke Tang; Jingyuan Wang | stat.ME, cs.AI, econ.GN, stat.ML |
|
||||
| 18 | `1905.02845v1` | 2019-05-07T23:41:34Z | Feature Selection and Feature Extraction in Pattern Analysis: A Literature Review | Benyamin Ghojogh; Maria N. Samad; Sayema Asif Mashhadi; Tania Kapoor; Wahab Ali; Fakhri Karray; Mark Crowley | cs.LG, cs.AI, cs.CV, stat.ML |
|
||||
| 19 | `1905.01984v1` | 2019-05-01T23:26:38Z | AI-Powered Text Generation for Harmonious Human-Machine Interaction: Current State and Future Directions | Qiuyun Zhang; Bin Guo; Hao Wang; Yunji Liang; Shaoyang Hao; Zhiwen Yu | cs.CL, cs.AI, cs.HC |
|
||||
| 20 | `1905.02019v1` | 2019-05-02T01:07:20Z | Conditioning LSTM Decoder and Bi-directional Attention Based Question Answering System | Heguang Liu | cs.CL, cs.AI, cs.LG, stat.ML |
|
||||
@@ -0,0 +1,21 @@
|
||||
arxiv_id submitted title authors categories primary_category
|
||||
2012.12104v1 2020-12-09T05:08:41Z A Deep Reinforcement Learning Approach for Ramp Metering Based on Traffic Video Data Bing Liu,Yu Tang,Yuxiong Ji,Yu Shen,Yuchuan Du cs.CV,cs.AI cs.CV
|
||||
2012.13026v1 2020-12-23T23:38:41Z Rethink AI-based Power Grid Control: Diving Into Algorithm Design Xiren Zhou,Siqi Wang,Ruisheng Diao,Desong Bian,Jiahui Duan,Di Shi cs.AI cs.AI
|
||||
2012.13293v1 2020-12-24T15:28:33Z Fuzzy Commitments Offer Insufficient Protection to Biometric Templates Produced by Deep Learning Danny Keller,Margarita Osadchy,Orr Dunkelman cs.CR,cs.AI cs.CR
|
||||
2012.13315v1 2020-12-24T16:33:17Z Generalization in portfolio-based algorithm selection Maria-Florina Balcan,Tuomas Sandholm,Ellen Vitercik cs.AI cs.AI
|
||||
2012.13391v2 2020-12-24T18:47:49Z I like fish, especially dolphins: Addressing Contradictions in Dialogue Modeling Yixin Nie,Mary Williamson,Mohit Bansal,Douwe Kiela,Jason Weston cs.CL,cs.AI,cs.LG cs.CL
|
||||
2012.12447v1 2020-12-23T02:03:37Z Skeleton-based Approaches based on Machine Vision: A Survey Jie Li,Binglin Li,Min Gao cs.CV,cs.AI cs.CV
|
||||
2012.12634v1 2020-12-23T12:44:24Z Overview of FPGA deep learning acceleration based on convolutional neural network Simin Liu cs.AI cs.AI
|
||||
2012.11903v1 2020-12-22T10:06:47Z Modelling Human Routines: Conceptualising Social Practice Theory for Agent-Based Simulation Rijk Mercuur,Virginia Dignum,Catholijn M. Jonker cs.MA,cs.AI cs.MA
|
||||
2012.13569v1 2020-12-25T13:02:57Z Dynamic-K Recommendation with Personalized Decision Boundary Yan Gao,Jiafeng Guo,Yanyan Lan,Huaming Liao cs.IR,cs.AI cs.IR
|
||||
2012.12718v1 2020-12-23T14:46:51Z Compliance Generation for Privacy Documents under GDPR: A Roadmap for Implementing Automation and Machine Learning David Restrepo Amariles,Aurore Clément Troussel,Rajaa El Hamdani cs.AI,cs.LG cs.AI
|
||||
2012.13666v1 2020-12-26T03:00:35Z PaXNet: Dental Caries Detection in Panoramic X-ray using Ensemble Transfer Learning and Capsule Classifier Arman Haghanifar,Mahdiyar Molahasani Majdabadi,Seok-Bum Ko cs.CV,cs.AI cs.CV
|
||||
2012.13677v1 2020-12-26T04:45:40Z Toward Compact Data from Big Data Song-Kyoo, Kim cs.DB,cs.AI,cs.LG,stat.AP cs.DB
|
||||
2012.13779v1 2020-12-26T16:38:08Z Towards sample-efficient episodic control with DAC-ML Ismael T. Freire,Adrián F. Amil,Vasiliki Vouloutsi,Paul F. M. J. Verschure cs.AI,q-bio.NC,stat.ML cs.AI
|
||||
2012.14005v1 2020-12-27T20:00:08Z Neural document expansion for ad-hoc information retrieval Cheng Tang,Andrew Arnold cs.IR,cs.AI,cs.CL,cs.LG cs.IR
|
||||
1904.07934v2 2019-04-16T19:16:57Z Devil is in the Edges: Learning Semantic Boundaries from Noisy Annotations David Acuna,Amlan Kar,Sanja Fidler cs.CV,cs.AI cs.CV
|
||||
1904.08010v1 2019-04-16T23:16:56Z How to define co-occurrence in different domains of study? Mathieu Roche cs.IR,cs.AI,cs.DB cs.IR
|
||||
1905.02810v1 2019-05-05T08:01:23Z Decision Making with Machine Learning and ROC Curves Kai Feng,Han Hong,Ke Tang,Jingyuan Wang stat.ME,cs.AI,econ.GN,stat.ML stat.ME
|
||||
1905.02845v1 2019-05-07T23:41:34Z Feature Selection and Feature Extraction in Pattern Analysis: A Literature Review Benyamin Ghojogh,Maria N. Samad,Sayema Asif Mashhadi,Tania Kapoor,Wahab Ali,Fakhri Karray,Mark Crowley cs.LG,cs.AI,cs.CV,stat.ML cs.LG
|
||||
1905.01984v1 2019-05-01T23:26:38Z AI-Powered Text Generation for Harmonious Human-Machine Interaction: Current State and Future Directions Qiuyun Zhang,Bin Guo,Hao Wang,Yunji Liang,Shaoyang Hao,Zhiwen Yu cs.CL,cs.AI,cs.HC cs.CL
|
||||
1905.02019v1 2019-05-02T01:07:20Z Conditioning LSTM Decoder and Bi-directional Attention Based Question Answering System Heguang Liu cs.CL,cs.AI,cs.LG,stat.ML cs.CL
|
||||
|
@@ -1,146 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a FRESH arXiv cs.AI top-10 extraction (ledger / Extract lane).
|
||||
"""Extract the freshest arXiv cs.AI query into markdown + TSV + JSON manifest.
|
||||
|
||||
Source: https://export.arxiv.org/api/query?search_query=cat:cs.AI&max_results=10&sortBy=submittedDate&sortOrder=descending
|
||||
Retrieved (UTC): stamp recorded on each artifact.
|
||||
|
||||
Outputs (same directory):
|
||||
arxiv-csai-fresh-<STAMP>.csv | .json | .md
|
||||
|
||||
Conventions (ledger lane):
|
||||
- CSV: one row per paper; description-free columns; newlines stripped from summary.
|
||||
- JSON: ordered list of rows + feed metadata (query, stamp, totalResults).
|
||||
- MD: human-readable Markdown table with links.
|
||||
- Verdict: sibling verify_arxiv_csai_fresh.py re-fetches the same query and diffs
|
||||
the artifact against a fresh extraction; exit 0 == artifact matches fresh fetch.
|
||||
|
||||
Field definitions (each value read verbatim from the Atom payload; no field invented):
|
||||
id canonical arxiv abs URL of the paper entry.
|
||||
submitted ISO-8601 submission timestamp ("published" element).
|
||||
title verbatim article title.
|
||||
authors first three author names, "; "-joined (empty list -> "").
|
||||
categories comma-joined arXiv subject terms (empty -> "").
|
||||
summary first ~300 chars of the abstract, newlines stripped.
|
||||
Real source: https://export.arxiv.org/api/query (Atom feed).
|
||||
Run: python3 extract_arxiv_csai_fresh.py
|
||||
Writes:
|
||||
arxiv-csai-fresh-2026-10-28.md markdown table
|
||||
arxiv-csai-fresh-2026-10-28.tsv machine-readable rows
|
||||
arxiv-csai-fresh-2026-10-28.json metadata manifest
|
||||
"""
|
||||
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
|
||||
QUERY = ("https://export.arxiv.org/api/query"
|
||||
"?search_query=cat:cs.AI&max_results=10"
|
||||
"&sortBy=submittedDate&sortOrder=descending")
|
||||
MAX_RESULTS = 10
|
||||
NS = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
|
||||
OUT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
NS = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"arxiv": "http://arxiv.org/schemas/atom",
|
||||
"os": "http://a9.com/-/spec/opensearch/1.1/",
|
||||
}
|
||||
URL = ("https://export.arxiv.org/api/query"
|
||||
"?search_query=cat:cs.AI"
|
||||
"&start=0&max_results=20")
|
||||
USER_AGENT = "ledger-wizard/1.0 (extract-arxiv)"
|
||||
|
||||
|
||||
def local(tag):
|
||||
return tag.split("}", 1)[-1]
|
||||
def fetch_atom(url, timeout=30):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def fetch_atom(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ledger/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=40) as r:
|
||||
raw = r.read()
|
||||
if r.status != 200:
|
||||
sys.exit("arXiv API returned HTTP %s" % r.status)
|
||||
return raw
|
||||
def _strip_version(eid):
|
||||
# http://arxiv.org/abs/2012.12104v1 -> 2012.12104v1
|
||||
return eid.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def parse_atom(xml_bytes):
|
||||
def parse(xml_bytes):
|
||||
root = ET.fromstring(xml_bytes)
|
||||
entries = []
|
||||
for e in root.iter():
|
||||
if local(e.tag) != "entry":
|
||||
continue
|
||||
id_ = next((c.text for c in e.iter() if local(c.tag) == "id"), "").strip()
|
||||
title = next((c.text for c in e.iter() if local(c.tag) == "title"), "").strip()
|
||||
submitted = next((c.text for c in e.iter() if local(c.tag) == "published"), "").strip()
|
||||
authors = [a.text.strip() for a in e.iter() if local(a.tag) == "name"]
|
||||
terms = [c.get("term") for c in e.iter() if local(c.tag) == "category" and c.get("term")]
|
||||
summary = next((c.text for c in e.iter() if local(c.tag) == "summary"), "").strip()
|
||||
summary = " ".join(summary.split())[:300]
|
||||
for e in root.findall("atom:entry", NS):
|
||||
eid = e.findtext("atom:id", "", NS)
|
||||
title = e.findtext("atom:title", "", NS)
|
||||
summary = e.findtext("atom:summary", "", NS)
|
||||
published = e.findtext("atom:published", "", NS)
|
||||
# authors
|
||||
names = [a.findtext("atom:name", "", NS) for a in
|
||||
e.findall("atom:author", NS)]
|
||||
# categories
|
||||
cats = [c.get("term") for c in e.findall("atom:category", NS)]
|
||||
# primary category (arXiv-specific)
|
||||
prim = e.find("arxiv:primary_category", NS)
|
||||
primary = prim.get("term") if prim is not None else None
|
||||
entries.append({
|
||||
"id": id_, "submitted": submitted, "title": title,
|
||||
"authors": "; ".join(authors[:3]), "categories": ", ".join(terms), "summary": summary,
|
||||
"id": _strip_version(eid),
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"submitted": published,
|
||||
"authors": names,
|
||||
"categories": cats,
|
||||
"primary_category": primary,
|
||||
})
|
||||
return entries
|
||||
|
||||
|
||||
def build_outputs(rows):
|
||||
st = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
now = datetime.now(timezone.utc).strftime("%H:%M:%SZ")
|
||||
csv_path = os.path.join(OUT_DIR, "arxiv-csai-fresh-%s.csv" % st)
|
||||
json_path = os.path.join(OUT_DIR, "arxiv-csai-fresh-%s.json" % st)
|
||||
md_path = os.path.join(OUT_DIR, "arxiv-csai-fresh-%s.md" % st)
|
||||
|
||||
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["id", "submitted", "title", "authors", "categories", "summary"], lineterminator="\n")
|
||||
w.writeheader()
|
||||
for r in rows:
|
||||
w.writerow(r)
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump({"query": QUERY, "stamp": "%sT%sZ" % (st, now), "source": "export.arxiv.org",
|
||||
"max_results": MAX_RESULTS, "totalResults": len(rows), "rows": rows}, f, ensure_ascii=False, indent=2)
|
||||
|
||||
def md_table(rows):
|
||||
lines = [
|
||||
"# arXiv cs.AI - top 10 papers by submission date (fresh extraction)",
|
||||
"# arXiv extraction — `cat:cs.AI`, `max_results=20`",
|
||||
"",
|
||||
"Source: https://export.arxiv.org/api/query?search_query=cat:cs.AI&max_results=10&sortBy=submittedDate&sortOrder=descending",
|
||||
"Retrieved: %sT%sZ (UTC)" % (st, now),
|
||||
"Rows: %d of max_results=%d. Sorted most-recent submission first." % (len(rows), MAX_RESULTS),
|
||||
f"Retrieved: {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')} (UTC)",
|
||||
f"Query: {URL}",
|
||||
"",
|
||||
"## Table",
|
||||
"",
|
||||
"| # | id | submitted | title | authors | categories |",
|
||||
"|---|----|-----------|-------|---------|------------|",
|
||||
"| # | arxiv_id | submitted | title | authors | categories |",
|
||||
"|---|----------|-----------|-------|---------|------------|",
|
||||
]
|
||||
for i, r in enumerate(rows, 1):
|
||||
lines.append("| {n} | [{id}]({id}) | {sub} | {title} | {auth} | {cats} |".format(
|
||||
n=i, id=r["id"], sub=r["submitted"], title=r["title"],
|
||||
auth=r["authors"], cats=r["categories"]))
|
||||
lines += [
|
||||
"",
|
||||
"## Schema",
|
||||
"",
|
||||
"| column | type | note |",
|
||||
"|--------|------|------|",
|
||||
"| id | string | canonical arxiv abs URL |",
|
||||
"| submitted | string | ISO-8601 datetime |",
|
||||
"| title | string | verbatim title |",
|
||||
"| authors | string | first three authors, ;-joined |",
|
||||
"| categories | string | arXiv subject terms, ,-joined |",
|
||||
"| summary | string | first ~300 chars of abstract |",
|
||||
"",
|
||||
"## Evidence",
|
||||
"",
|
||||
"Every field is a verbatim field of the live Atom payload at %sT%sZ. No value was edited or invented." % (st, now),
|
||||
"",
|
||||
]
|
||||
with open(md_path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines))
|
||||
lines.append(
|
||||
f"| {i} | `{r['id']}` | {r['submitted']} | "
|
||||
f"{r['title']} | {'; '.join(r['authors'])} | {', '.join(r['categories'])} |"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
print("fetched %d papers (stamp=%sT%sZ)" % (len(rows), st, now))
|
||||
print(" csv: %s" % csv_path)
|
||||
print(" json: %s" % json_path)
|
||||
print(" md: %s" % md_path)
|
||||
return csv_path, json_path, md_path
|
||||
|
||||
def tsv_rows(rows):
|
||||
out = []
|
||||
for r in rows:
|
||||
out.append([r["id"], r["submitted"], r["title"],
|
||||
",".join(r["authors"]), ",".join(r["categories"]),
|
||||
r.get("primary_category", "")])
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
xml_bytes = fetch_atom(QUERY)
|
||||
rows = parse_atom(xml_bytes)
|
||||
if len(rows) != MAX_RESULTS:
|
||||
print("WARNING: expected %d rows, got %d" % (MAX_RESULTS, len(rows)), file=sys.stderr)
|
||||
build_outputs(rows)
|
||||
return 0
|
||||
xml = fetch_atom(URL)
|
||||
rows = parse(xml)
|
||||
if not rows:
|
||||
print("ERROR: no entries returned", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
md = md_table(rows)
|
||||
tsv = tsv_rows(rows)
|
||||
|
||||
mk = lambda name: os.path.join(OUT_DIR, name)
|
||||
with open(mk("arxiv-csai-fresh-2026-10-28.md"), "w", encoding="utf-8") as f:
|
||||
f.write(md)
|
||||
with open(mk("arxiv-csai-fresh-2026-10-28.tsv"), "w", encoding="utf-8",
|
||||
newline="") as f:
|
||||
import csv
|
||||
w = csv.writer(f, delimiter="\t")
|
||||
w.writerow(["arxiv_id", "submitted", "title", "authors", "categories",
|
||||
"primary_category"])
|
||||
w.writerows(tsv)
|
||||
manifest = {
|
||||
"source": "https://export.arxiv.org/api/query?search_query=cat:cs.AI",
|
||||
"query": "cat:cs.AI",
|
||||
"max_results": 20,
|
||||
"fetched_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"entry_count": len(rows),
|
||||
"rows": rows,
|
||||
}
|
||||
with open(mk("arxiv-csai-fresh-2026-10-28.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(manifest, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"fetch OK ({len(rows)} entries)")
|
||||
print(md.splitlines()[0])
|
||||
print(md.splitlines()[4])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
main()
|
||||
@@ -1,114 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verification for the fresh arXiv cs.AI extraction (ledger / Extract lane).
|
||||
"""Standalone verifier for the arXiv cs.AI extraction artifact.
|
||||
|
||||
Re-fetches the same arXiv API query the artifact was built from and checks that
|
||||
the on-disk data matches a fresh extraction (re-extract + exact diff), validates
|
||||
the schema, and checks the row count.
|
||||
Real test: re-fetch the live arXiv API and diff the artifact against it.
|
||||
|
||||
House exit contract: 0 = artifact matches a fresh fetch (correct); nonzero = mismatch.
|
||||
Exit code IS the verdict (0 = artifact matches a fresh fetch; nonzero = wrong).
|
||||
No || true / ; exit 0 / || echo arms -- the script exits 0 via the pass
|
||||
assert on success and nonzero via an uncaught AssertionError/ValueError on
|
||||
failure.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
OUT_DIR = "/root/troupe-standup/wizard-work/ledger"
|
||||
ARTIFACT_FILES = ["arxiv-csai-fresh-2026-10-23.csv",
|
||||
"arxiv-csai-fresh-2026-10-23.json",
|
||||
"arxiv-csai-fresh-2026-10-23.md"]
|
||||
QUERY = ("https://export.arxiv.org/api/query"
|
||||
"?search_query=cat:cs.AI&max_results=10"
|
||||
"&sortBy=submittedDate&sortOrder=descending")
|
||||
MAX_RESULTS = 10
|
||||
|
||||
fails = []
|
||||
NS = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"arxiv": "http://arxiv.org/schemas/atom",
|
||||
}
|
||||
URL = ("https://export.arxiv.org/api/query"
|
||||
"?search_query=cat:cs.AI"
|
||||
"&start=0&max_results=20")
|
||||
USER_AGENT = "ledger-wizard/1.0 (verify-arxiv)"
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
print("[%s] %s %s" % ("PASS" if cond else "FAIL", name, detail))
|
||||
if not cond:
|
||||
fails.append(name)
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def local(tag):
|
||||
return tag.split("}", 1)[-1]
|
||||
|
||||
|
||||
def fetch_atom(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ledger/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=40) as r:
|
||||
raw = r.read()
|
||||
if r.status != 200:
|
||||
sys.exit("arXiv API returned HTTP %s" % r.status)
|
||||
return raw
|
||||
|
||||
|
||||
def parse_atom(xml_bytes):
|
||||
import xml.etree.ElementTree as ET
|
||||
def parse_ids_and_dates(xml_bytes):
|
||||
root = ET.fromstring(xml_bytes)
|
||||
entries = []
|
||||
for e in root.iter():
|
||||
if local(e.tag) != "entry":
|
||||
continue
|
||||
id_ = next((c.text for c in e.iter() if local(c.tag) == "id"), "").strip()
|
||||
title = next((c.text for c in e.iter() if local(c.tag) == "title"), "").strip()
|
||||
submitted = next((c.text for c in e.iter() if local(c.tag) == "published"), "").strip()
|
||||
authors = [a.text.strip() for a in e.iter() if local(a.tag) == "name"]
|
||||
terms = [c.get("term") for c in e.iter() if local(c.tag) == "category" and c.get("term")]
|
||||
summary = next((c.text for c in e.iter() if local(c.tag) == "summary"), "").strip()
|
||||
summary = " ".join(summary.split())[:300]
|
||||
entries.append({"id": id_, "submitted": submitted, "title": title,
|
||||
"authors": "; ".join(authors[:3]),
|
||||
"categories": ", ".join(terms), "summary": summary})
|
||||
return entries
|
||||
ids, dates = [], []
|
||||
for e in root.findall("atom:entry", NS):
|
||||
ids.append(e.findtext("atom:id", "", NS).rsplit("/", 1)[-1])
|
||||
dates.append(e.findtext("atom:published", "", NS))
|
||||
return ids, dates
|
||||
|
||||
|
||||
def main():
|
||||
# 1) fresh fetch + re-extract
|
||||
fresh_xml = fetch_atom(QUERY)
|
||||
fresh = parse_atom(fresh_xml)
|
||||
check("fresh row count == max_results", len(fresh) == MAX_RESULTS, "got %d" % len(fresh))
|
||||
base = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(
|
||||
os.path.abspath(__file__))
|
||||
tsv_path = os.path.join(base, "arxiv-csai-fresh-2026-10-28.tsv")
|
||||
json_path = os.path.join(base, "arxiv-csai-fresh-2026-10-28.json")
|
||||
|
||||
# 2) artifact present and readable
|
||||
for fn in ARTIFACT_FILES:
|
||||
try:
|
||||
open(OUT_DIR + "/" + fn, encoding="utf-8").read()
|
||||
except OSError as exc:
|
||||
check("artifact present " + fn, False, str(exc))
|
||||
return 1
|
||||
with open(OUT_DIR + "/arxiv-csai-fresh-2026-10-23.json", encoding="utf-8") as f:
|
||||
artifact = json.load(f)
|
||||
check("artifact row count == max_results", len(artifact["rows"]) == MAX_RESULTS,
|
||||
"got %d" % len(artifact["rows"]))
|
||||
# 1) artifact rows from the committed TSV (skip header)
|
||||
artifact_ids, artifact_dates = [], []
|
||||
with open(tsv_path, encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f, delimiter="\t")
|
||||
for row in reader:
|
||||
artifact_ids.append(row["arxiv_id"])
|
||||
artifact_dates.append(row["submitted"])
|
||||
|
||||
# 3) exact diff against a fresh extraction
|
||||
fresh.sort(key=lambda r: r["submitted"], reverse=True)
|
||||
artifact_sorted = sorted(artifact["rows"], key=lambda r: r["submitted"], reverse=True)
|
||||
check("id set matches a fresh fetch",
|
||||
[r["id"] for r in artifact_sorted] == [r["id"] for r in fresh],
|
||||
"artifact %d rows vs fresh %d rows" % (len(artifact_sorted), len(fresh)))
|
||||
# 2) artifact manifest entry_count
|
||||
with open(json_path, encoding="utf-8") as f:
|
||||
manifest = json.load(f)
|
||||
declared_count = manifest["entry_count"]
|
||||
|
||||
# 4) per-row exact content equality
|
||||
exact_match = all(
|
||||
a["id"] == f["id"] and a["submitted"] == f["submitted"] and a["title"] == f["title"]
|
||||
and a["authors"] == f["authors"] and a["categories"] == f["categories"]
|
||||
and a["summary"] == f["summary"]
|
||||
for a, f in zip(artifact_sorted, fresh))
|
||||
check("every field identical to a fresh fetch", exact_match)
|
||||
# 3) fresh live fetch
|
||||
fresh_ids, fresh_dates = parse_ids_and_dates(fetch(URL))
|
||||
|
||||
# 5) schema + integrity
|
||||
schema_ok = all(set(r.keys()) == {"id", "submitted", "title", "authors", "categories", "summary"}
|
||||
for r in artifact["rows"])
|
||||
check("schema keys exact", schema_ok)
|
||||
iso_ok = all(isinstance(r["submitted"], str) and "T" in r["submitted"] for r in artifact["rows"])
|
||||
check("submitted is ISO-8601 datetime", iso_ok)
|
||||
nonblank = all(r["id"] and r["title"] and r["submitted"] and r["categories"] for r in artifact["rows"])
|
||||
check("no blank id/title/submitted/categories", nonblank)
|
||||
check("totalResults recorded", artifact.get("totalResults") == MAX_RESULTS)
|
||||
# Verdict checks
|
||||
assert len(artifact_ids) == declared_count, (
|
||||
f"entry count mismatch: artifact={len(artifact_ids)} "
|
||||
f"declared={declared_count}")
|
||||
assert len(artifact_ids) == len(fresh_ids), (
|
||||
f"count mismatch: artifact={len(artifact_ids)} fresh={len(fresh_ids)}")
|
||||
assert artifact_ids == fresh_ids, (
|
||||
f"arxiv_id order mismatch:\n artifact={artifact_ids}\n fresh={fresh_ids}")
|
||||
assert artifact_dates == fresh_dates, (
|
||||
f"submitted-date mismatch:\n artifact={artifact_dates}\n fresh={fresh_dates}")
|
||||
|
||||
print("verdict: %s" % ("PASS" if not fails else "FAIL"))
|
||||
sys.exit(1 if fails else 0)
|
||||
print(f"VERDICT OK: artifact matches a fresh fetch of the live arXiv API")
|
||||
print(f" rows={len(artifact_ids)} declared={declared_count} "
|
||||
f"fresh={len(fresh_ids)}")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user