0

Cannot to retrieve contet from NewYorkTimes by using "select html" query

Hello, I am implementing web scrapper by using YQL. I use simple query "select from html where url='xxxx' " in order to retrieve page content with xml format. This works for many sites but some sites does not give me any contents back and I am not sure how I can debug the problem.

One of the examples is an article from New York Times, "http://www.nytimes.com/2013/05/20/movies/coen-brothers-inside-llewyn-davis-is-popular-at-cannes.html" when I used this YQL query and get empty result.

I enabled diagnostic and get the following result seeing lots of redirect and cache tags. Could anyone help to point out what is wrong with my query?

Thanks!

<?xml version="1.0" encoding="UTF-8"?>
<query <xmlns:yahoo=>"<http://www.yahooapis.com/v1/base.rng>"
    <yahoo:count=>"0" <yahoo:created=>"2013-05-20T00:12:17Z" <yahoo:lang=>"en-US">
    <diagnostics>
        <publiclyCallable>true</publiclyCallable>
        <cache execution-start-time="1" execution-stop-time="1"
            execution-time="0" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D0&OP=744f910bQ2FQ2B-1iQ2Bt8iQ2BQ60Q60Q60Q2BQ3EicQ20Q2Bx-jQ5Eh--iQ2FQ2BQ2Fa!EQ2BaeQ2BQ2FaQ2BQ60>-hQ20xQ2BczxxQ20LL.Q5EiQ2BQ5E8hz.tR.hc8Rc-WLQ5ERi-RhLQ23LQ20RQ3ELQ20xRQ24Q2AQ5E.8hVQ3EicQ20]]></redirect>
        <cache execution-start-time="119" execution-stop-time="120"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D0&amp;OP=744f910bQ2FQ2B-1iQ2Bt8iQ2BQ60Q60Q60Q2BQ3EicQ20Q2Bx-jQ5Eh--iQ2FQ2BQ2Fa!EQ2BaeQ2BQ2FaQ2BQ60>-hQ20xQ2BczxxQ20LL.Q5EiQ2BQ5E8hz.tR.hc8Rc-WLQ5ERi-RhLQ23LQ20RQ3ELQ20xRQ24Q2AQ5E.8hVQ3EicQ20" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=0>]]></redirect>
        <cache execution-start-time="218" execution-stop-time="219"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=0>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D1Q26&OP=57d6b1feQ2F3Q3Am>(3Q7Be(3...32(9x35Q3ABQ5BZQ3AQ3A(Q7D3Q7DR!C3Ry3Q7DR3.Q3AZx539655xXX0Q5B(3Q5BeZ60Q7BE0Z9eE9Q3AbXQ5BE(Q3AEZX-XxE2Xx5ENsQ5B0eZh2(9x]]></redirect>
        <cache execution-start-time="279" execution-stop-time="280"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D1Q26&amp;OP=57d6b1feQ2F3Q3Am>(3Q7Be(3...32(9x35Q3ABQ5BZQ3AQ3A(Q7D3Q7DR!C3Ry3Q7DR3.Q3AZx539655xXX0Q5B(3Q5BeZ60Q7BE0Z9eE9Q3AbXQ5BE(Q3AEZX-XxE2Xx5ENsQ5B0eZh2(9x" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=1&>]]></redirect>
        <cache execution-start-time="418" execution-stop-time="418"
            execution-time="0" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=1&amp;>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D2Q26&OP=831e879eQ2FtUK4tQ3BQ234tLLLtB4V_tMUQ5ByrUU4Q3DtQ3D-bdt-1tQ3D-tLUr_MtVQ5EMM_99Q5Cy4tyQ23rQ5EQ5CQ3B@Q5CrVQ23@VUp9y@4U@r9x9_@B9_M@Q7BFyQ5CQ23rYB4V_>]]></redirect>
        <cache execution-start-time="477" execution-stop-time="478"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D2Q26&amp;OP=831e879eQ2FtUK4tQ3BQ234tLLLtB4V_tMUQ5ByrUU4Q3DtQ3D-bdt-1tQ3D-tLUr_MtVQ5EMM_99Q5Cy4tyQ23rQ5EQ5CQ3B@Q5CrVQ23@VUp9y@4U@r9x9_@B9_M@Q7BFyQ5CQ23rYB4V_>" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=2&>]]></redirect>
        <cache execution-start-time="616" execution-stop-time="617"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=2&amp;>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D3Q26&OP=be027bf9Q2FoQ23DJoQ7DnJo@@@oQ51JjYoXQ23EWkQ23Q23JQ5DoQ5D5Q25Q27o5woQ5D5o@Q23kYXojfXXYKKPWJoWnkfPQ7DAPkjnAjQ230KWAJQ23AkKBKYAQ51KYXAV6WPnkQ5EQ51JjY>]]></redirect>
        <cache execution-start-time="680" execution-stop-time="680"
            execution-time="0" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D3Q26&amp;OP=be027bf9Q2FoQ23DJoQ7DnJo@@@oQ51JjYoXQ23EWkQ23Q23JQ5DoQ5D5Q25Q27o5woQ5D5o@Q23kYXojfXXYKKPWJoWnkfPQ7DAPkjnAjQ230KWAJQ23AkKBKYAQ51KYXAV6WPnkQ5EQ51JjY>" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=3&>]]></redirect>
        <cache execution-start-time="818" execution-stop-time="818"
            execution-time="0" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=3&amp;>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D4Q26&OP=6f0a2bcbQ2FQ60rhyQ60s3yQ60444Q60fy>(Q2AQ60,rv1krry9Q6098)Q7CQ608qQ6098Q604rkQ2A,Q60(n,,Q2AQ5EQ5EQ5C1yQ6013knQ5CsQ24Q5Ck(3Q24(rQ2FQ5E1Q24yrQ24kQ5EMQ5EQ2AQ24fQ5EQ2A,Q24bo1Q5C3kQ2Bfy(Q2A]]></redirect>
        <cache execution-start-time="882" execution-stop-time="883"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D4Q26&amp;OP=6f0a2bcbQ2FQ60rhyQ60s3yQ60444Q60fy>(Q2AQ60,rv1krry9Q6098)Q7CQ608qQ6098Q604rkQ2A,Q60(n,,Q2AQ5EQ5EQ5C1yQ6013knQ5CsQ24Q5Ck(3Q24(rQ2FQ5E1Q24yrQ24kQ5EMQ5EQ2AQ24fQ5EQ2A,Q24bo1Q5C3kQ2Bfy(Q2A" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=4&>]]></redirect>
        <cache execution-start-time="1022" execution-stop-time="1023"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <url error="Connect Failure" execution-start-time="1"
            execution-stop-time="1082" execution-time="1081" proxy="DEFAULT"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp>]]></url>
        <cache execution-start-time="1082" execution-stop-time="1083"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D0&OP=7e0a1777Q2FQ7DQ7ElOQ7Do.OQ7DgggQ7DQ24OFqQ7DzQ7EQ2BPhQ7EQ7EOKQ7DKEQ20MQ7DEQ5BQ7DKEQ7DgQ7EhqzQ7DFvzzqQ25Q25Q3CPOQ7DP.hvQ3CofQ3ChF.fFQ7EYQ25PfOQ7EfhQ25cQ25qfQ24Q25qzfjpPQ3C.hXQ24OFq>]]></redirect>
        <cache execution-start-time="1205" execution-stop-time="1206"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D0&amp;OP=7e0a1777Q2FQ7DQ7ElOQ7Do.OQ7DgggQ7DQ24OFqQ7DzQ7EQ2BPhQ7EQ7EOKQ7DKEQ20MQ7DEQ5BQ7DKEQ7DgQ7EhqzQ7DFvzzqQ25Q25Q3CPOQ7DP.hvQ3CofQ3ChF.fFQ7EYQ25PfOQ7EfhQ25cQ25qfQ24Q25qzfjpPQ3C.hXQ24OFq>" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=0>]]></redirect>
        <cache execution-start-time="1304" execution-stop-time="1305"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=0>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D1Q26&OP=41ae2d94Q2FQ27za>,Q27iB,Q27sssQ27Q51,CQ2BQ271z9GQ7Dzz,nQ27nTS5Q27T3Q27nTQ27szQ7DQ2B1Q27CP11Q2BUUQ7CG,Q27GBQ7DPQ7Ci!Q7CQ7DCB!Cz.UG!,z!Q7DUQ2FUQ2B!Q51UQ2B1!KuGQ7CBQ7D2Q51,CQ2B]]></redirect>
        <cache execution-start-time="1369" execution-stop-time="1370"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D1Q26&amp;OP=41ae2d94Q2FQ27za>,Q27iB,Q27sssQ27Q51,CQ2BQ271z9GQ7Dzz,nQ27nTS5Q27T3Q27nTQ27szQ7DQ2B1Q27CP11Q2BUUQ7CG,Q27GBQ7DPQ7Ci!Q7CQ7DCB!Cz.UG!,z!Q7DUQ2FUQ2B!Q51UQ2B1!KuGQ7CBQ7D2Q51,CQ2B" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=1&>]]></redirect>
        <cache execution-start-time="1468" execution-stop-time="1469"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=1&amp;>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D2Q26&OP=fb92df2aQ2F@Q25gi@Zti@kkk@JiWQ5D@Q2BQ25RQ26>(<Q25Q25iQ51@Q51-eU@-Q5B@Q51-@kQ25>(<Q5DQ2B@WQ5EQ2BQ2BQ5DLLdQ26i@Q26t>(Q5EdZPd(WtPWQ25.LQ26PiQ25P(LXLQ5DPJLQ5DQ2BPcQ23Q26dt(3JiWQ5D]]></redirect>
        <cache execution-start-time="1529" execution-stop-time="1530"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D2Q26&amp;OP=fb92df2aQ2F@Q25gi@Zti@kkk@JiWQ5D@Q2BQ25RQ26>(<Q25Q25iQ51@Q51-eU@-Q5B@Q51-@kQ25>(<Q5DQ2B@WQ5EQ2BQ2BQ5DLLdQ26i@Q26t>(Q5EdZPd(WtPWQ25.LQ26PiQ25P(LXLQ5DPJLQ5DQ2BPcQ23Q26dt(3JiWQ5D" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=2&>]]></redirect>
        <cache execution-start-time="1628" execution-stop-time="1629"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=2&amp;>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D3Q26&OP=94e471fdQ2FcnjIcbRIcrrrcJILTcynQ51@VnnIQ23cQ23vQ7DQ22cvAcQ23vcrnVTycLlyyT88Q5B@Ic@RVlQ5BbUQ5BVLRULnd8@UInUV8i8TUJ8TyUoQ20@Q5BRVMJILT>]]></redirect>
        <cache execution-start-time="1694" execution-stop-time="1695"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D3Q26&amp;OP=94e471fdQ2FcnjIcbRIcrrrcJILTcynQ51@VnnIQ23cQ23vQ7DQ22cvAcQ23vcrnVTycLlyyT88Q5B@Ic@RVlQ5BbUQ5BVLRULnd8@UInUV8i8TUJ8TyUoQ20@Q5BRVMJILT>" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=3&>]]></redirect>
        <cache execution-start-time="1793" execution-stop-time="1794"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&amp;_r=3&amp;>" status="303"><![CDATA[<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&OQ=hpQ26_rQ3D4Q26&OP=424a60a3Q2FXnoQ60XkQ2BQ60XzzzXQ7EQ60dQ23XQ25nQ5ChQ3AnnQ600X0m2HXmbX0mXznQ3AQ23Q25Xd.Q25Q25Q23KK_hQ60XhQ2BQ3A._k>)_Q3AdQ2B)dnqKh)Q60n)<Q3AK@KQ23>)Q7EKQ23Q25)!Q7Ch_Q2BQ3AwQ7EQ60dQ23]]></redirect>
        <cache execution-start-time="1855" execution-stop-time="1856"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <redirect
            from="<http://www.nytimes.com/glogin?URI=http>://<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html&amp;OQ=hpQ26_rQ3D4Q26&amp;OP=424a60a3Q2FXnoQ60XkQ2BQ60XzzzXQ7EQ60dQ23XQ25nQ5ChQ3AnnQ600X0m2HXmbX0mXznQ3AQ23Q25Xd.Q25Q25Q23KK_hQ60XhQ2BQ3A._k>)_Q3AdQ2B)dnqKh)Q60n)<Q3AK@KQ23>)Q7EKQ23Q25)!Q7Ch_Q2BQ3AwQ7EQ60dQ23" status="302"><![CDATA[<http://www.nytimes.com/2013/05/20/world/middleeast/syrian-army-moves-to-rebel-held-qusayr.html?hp&_r=4&>]]></redirect>
        <cache execution-start-time="1954" execution-stop-time="1955"
            execution-time="1" method="GET" type="MEMCACHED"><![CDATA[<http://www.nytimes.com/robots.txt>]]></cache>
        <user-time>2050</user-time>
        <service-time>1099</service-time>
        <build-version>36794</build-version>
    </diagnostics> 
    <results/>
</query>

by
0 Replies

Recent Posts

in YQL